:W&B8M qvnjquy8; 9 /0.28.1+bc2cf24b1bd9fa3a1b6cbeb920087b832afc526f0.70.0" lpoyjnswmzda qvnjquy8 qvnjquy8spatop" _wandb{}Bspatop_v6_blocks_outputjdp-spatop-train-v6-blocks-mkt2j եҼ1 (Hh"3.12.10*0.28.1j linux-x86_64؆\c -Linux-6.8.0-134-generic-x86_64-with-glibc2.36CPython 3.12.10 ե̃*-of*A/data/spatop/options_files/sweep_fixed_dp/config_gs8pex8v_v6.json*-l*/scratch/logs/*-n*spatop_v6_blocks_output2-m spanet.trainRdanielprimosch@gmail.comZ/scratch/logs/bdp-spatop-train-v6-blocks-mkt2jr/opt/conda/bin/python 6x4fd81gidj3xoyo6o3b68z9epn61c81W2  requirements.txtAJP qvnjquy8Z? (H (Hh"3.12.10*0.28.1j linux-x86_64xjR qvnjquy8ZA (H (H8h"3.12.10*0.28.1j linux-x86_64* qvnjquy8b trainer/global_step2;QU qvnjquy8ZD (H (H8h"3.12.10*0.28.1j linux-x86_64&]/ qvnjquy8b*"trainer/global_step2\:J qvnjquy8j9 ץ|*LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0] ' @@NVIDIA A100 80GB PCIe /7S NVIDIA A100 80GB PCIe6"Ampere*(GPU-1c79f124-b049-1e3d-7f6d-104357f4ddb613.2 6x4fd81gidj3xoyo6o3b68z9epn61c81 ͍ qvnjquy8j  ץ※ ┏━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━┳━━━━━━━┓ ┃ ┃ Name ┃ Type ┃ Params ┃ Mode ┃ FLOPs ┃ ┡━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━╇━━━━━━━┩ │ 0 │ embedding │ MultiInputVectorEmbed… │ 220 K │ train │ 0 │ │ 1 │ encoder │ JetEncoderWithPairwise │ 332 K │ train │ 0 │ │ 2 │ branch_decoders │ ModuleList │ 4.8 M │ train │ 0 │ │ 3 │ regression_decoder │ RegressionDecoder │ 0 │ train │ 0 │ │ 4 │ classification_decoder │ ClassificationDecoder │ 0 │ train │ 0 │ │ │ other params │ n/a │ 12 │ n/a │ n/a │ └───┴────────────────────────┴────────────────────────┴────────┴───────┴───────┘ @1\ qvnjquy8j ץ܄Trainable params: 5.3 M Non-trainable params: 50 Total params: 5.3 M Total estimated model params size (MB): 21.374 Modules in train mode: 2326 Modules in eval mode: 0 Total FLOPs: 0 Ɓ qvnjquy8*  hidden_dim64  transformer_dim64  transformer_dim_scale2.0  initial_embedding_dim64  position_embedding_dim64  num_embedding_layers10  num_encoder_layers4 ! num_branch_embedding_layers3  num_branch_encoder_layers3  num_jet_embedding_layers0  num_jet_encoder_layers2  num_detector_layers2  num_regression_layers3  num_classification_layers3  split_symmetric_attention1  num_attention_heads8 ! transformer_activation"gelu"  skip_connections1 ( "initial_embedding_skip_connections1  linear_block_type"GRU"  transformer_type"Gated"  linear_activation"gelu"  normalization "LayerNorm"  masking "Filling" linear_prelu_activationtrue  use_pairwise_interactions1  num_pairwise_features4  pairwise_embedding_dim8  pairwise_input_source"all"  pairwise_block_embeddings1  pairwise_cross_typetrue  use_moefalse  num_experts8  num_experts_per_tok2  moe_loss_scale0.01 K event_info_file7"/data/spatop/event_files/v11/tt_hadronic_v7_full.yaml" K training_file9"/data/spatop/delphes_v6/tt_hadronic_training2k_clean.h5"  validation_file""  testing_file""  normalize_features1  limit_to_num_jets0  balance_particles0  balance_jets0 ! balance_classificationsfalse  partial_events1  dataset_limit1.0  dataset_randomization0  train_validation_split0.95  limit_val_batches1.0  batch_size512  num_dataloader_workers0  mask_sequence_vectors1  combine_pair_loss"min"  optimizer"AdamW" ' learning_rate0.0007344299982717679  focal_gamma0.0  combinatorial_scale0.0 # learning_rate_warmup_epochs1.0  learning_rate_cycles1  assignment_loss_scale1.0  detection_loss_scale1.0  kl_loss_scale0.0  regression_loss_scale0.0 ! classification_loss_scale0.0 ! use_mass_classificationfalse  mass_classes[]  balance_lossestrue  l2_penalty0.0002  gradient_clip10.0  dropout0.2978723651141102 epochs10 num_gpu1  verbose_outputfalse  usable_gpus""  trial_time"" $ trial_output_dir"./test_output"uR qvnjquy8j ץ/opt/conda/lib/python3.12/site-packages/pytorch_lightning/utilities/_pytree.py:21: `isinstance(treespec, LeafSpec)` is deprecated, use `isinstance(treespec, TreeSpec) and treespec.is_leaf()` instead. BgN qvnjquy8j ץ/opt/conda/lib/python3.12/site-packages/pytorch_lightning/trainer/connectors/data_connector.py:434: The 'val_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=7` in the `DataLoader` to improve performance. Oe qvnjquy8j ٥/opt/conda/lib/python3.12/site-packages/pytorch_lightning/utilities/_pytree.py:21: `isinstance(treespec, LeafSpec)` is deprecated, use `isinstance(treespec, TreeSpec) and treespec.is_leaf()` instead. )P qvnjquy8j ٥/opt/conda/lib/python3.12/site-packages/pytorch_lightning/trainer/connectors/data_connector.py:434: The 'train_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=7` in the `DataLoader` to improve performance. yi(: ̙U gpu.0.gpu31.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp31.0 gpu.0.powerWatts122.447' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent40.815666666666665 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0Vvv"(:  W" proc.memory.rssMB 11260.359375* disk./.usagePercent32.343230778362944 disk.drbd1510.out0 cpu18.905988262182507 disk.rbd0.in0* proc.memory.percent17.181944847106934 disk.drbd2111.in0 disk.drbd2041.in0 disk.dm-0.out0.34375 disk.drbd1226.out0 disk.drbd2078.in0 disk.drbd1584.in0 disk.drbd2118.in0 disk.drbd1817.in0 disk.drbd1673.in0 disk.drbd1673.out0 disk.drbd1289.out0 disk.sda.out 0.50390625 disk.drbd2078.out0 proc.cpu.threads29 network.recv41970 disk.drbd1828.in0 disk.dm-0.in0 disk.nvme0n1.out0.34375$ memory_percent41.20677709579468 disk.drbd1817.out0 disk.drbd1584.out0$ disk./.usageGB578.3298149108887 disk.drbd1931.out0 disk.drbd1938.in0 disk.rbd0.out0 disk.drbd1356.out0 disk.drbd1510.in0 disk.drbd2118.out0 disk.sda2.out 0.50390625 disk.drbd1931.in0 disk.drbd1552.in0 disk.drbd1550.in0 disk.drbd1289.in0 disk.sda2.in0 disk.drbd1938.out0 disk.sda1.out0 disk.drbd1226.in0 disk.drbd1828.out0 disk.drbd2111.out0 disk.rbd1.out0 disk.drbd1552.out0 disk.sda.in0 disk.drbd1285.in0) proc.memory.availableMB 38530.7265625 network.sent43075 disk.drbd1965.out0 disk.drbd2041.out0 disk.sda1.in0 disk.drbd1356.in0 disk.drbd1550.out0 disk.drbd1285.out0 disk.rbd1.in0 disk.drbd1965.in0 disk.nvme0n1.in0%93b/ lr-AdamW/pg1"trainer/global_step2:JP-s`3b/ lr-AdamW/pg2"trainer/global_step2:JPP-) _wandbruntime26  _runtime26}` ' lr-AdamW/pg11.2628371377799987e-06 ' lr-AdamW/pg21.2628371377799987e-06 trainer/global_step49 % _timestamp1.7857953141064043e+09 _runtime 26.929535822 _step0i ' lr-AdamW/pg11.2628371377799987e-06 ' lr-AdamW/pg21.2628371377799987e-06 trainer/global_step49 % _timestamp1.7857953141064043e+09 _runtime 26.929535822 _step0\Sqbm JDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current"trainer/global_step2:JP=Jbb^ ;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak"trainer/global_step2:JPnbj GDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated"trainer/global_step2:JP7Pjbf CDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed"trainer/global_step2:JP{obk HDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current"trainer/global_step2:JPP*,qbm JDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current"trainer/global_step2:JPg bc @DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed"trainer/global_step2:JP?o!bk HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed"trainer/global_step2:JP1Lj"bf CDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current"trainer/global_step2:JPKr#bn KDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated"trainer/global_step2:JP^k$bg DDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated"trainer/global_step2:JP wj%bf CDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated"trainer/global_step2:JPue&ba >DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak"trainer/global_step2:JP;j'bf CDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed"trainer/global_step2:JP t(bp MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed"trainer/global_step2:JPB]e)ba >DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak"trainer/global_step2:JP.^k*bg DDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated"trainer/global_step2:JP|@d+b` =DeviceStatsMonitor.on_train_batch_start/segment.all.allocated"trainer/global_step2:JP5adh,bd ADeviceStatsMonitor.on_train_batch_start/active.large_pool.current"trainer/global_step2:JP5n-bj GDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current"trainer/global_step2:JPi:!f.bb ?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak"trainer/global_step2:JP,uW/bS 0DeviceStatsMonitor.on_train_batch_start/num_ooms"trainer/global_step2:JP f0bb ?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed"trainer/global_step2:JP¹n1bj GDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak"trainer/global_step2:JPo2l2bh EDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak"trainer/global_step2:JPuf3bb ?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed"trainer/global_step2:JP s4bo LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated"trainer/global_step2:JPA k5bg DDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated"trainer/global_step2:JPQ^ui6be BDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current"trainer/global_step2:JPFh7bd ADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed"trainer/global_step2:JPi8be BDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak"trainer/global_step2:JPB`9b\ 9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries"trainer/global_step2:JPl:bh EDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated"trainer/global_step2:JP@Ji;be BDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak"trainer/global_step2:JP nbj GDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed"trainer/global_step2:JPz,k?bg DDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak"trainer/global_step2:JPg@bc @DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed"trainer/global_step2:JPosAbo LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated"trainer/global_step2:JP cpBbl IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated"trainer/global_step2:JP!6doCbk HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed"trainer/global_step2:JP%]sDbo LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak"trainer/global_step2:JP+!"qEbm JDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current"trainer/global_step2:JP|+"pFbl IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated"trainer/global_step2:JP9mGbi FDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak"trainer/global_step2:JP?_Hb[ 8DeviceStatsMonitor.on_train_batch_start/active.all.freed"trainer/global_step2:JP"]IbY 6DeviceStatsMonitor.on_train_batch_start/max_split_size"trainer/global_step2:JP ] mJbi FDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated"trainer/global_step2:JPp2hKbd ADeviceStatsMonitor.on_train_batch_start/active.small_pool.current"trainer/global_step2:JPaLb] :DeviceStatsMonitor.on_train_batch_start/num_oom_rejections"trainer/global_step2:JP#`^MbZ 7DeviceStatsMonitor.on_train_batch_start/active.all.peak"trainer/global_step2:JP2nNbj GDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated"trainer/global_step2:JPq$^ObZ 7DeviceStatsMonitor.on_train_batch_start/num_device_free"trainer/global_step2:JP8DeviceStatsMonitor.on_train_batch_start/allocation.all.current"trainer/global_step2:JP'xgbt QDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated"trainer/global_step2:JPQohbk HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed"trainer/global_step2:JP3cib_ DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak"trainer/global_step2:JP)g|bc @DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed"trainer/global_step2:JPPa}b] :DeviceStatsMonitor.on_train_batch_start/active.all.current"trainer/global_step2:JP'j~bf CDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated"trainer/global_step2:JP?l"lbh EDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed"trainer/global_step2:JP&qbl IDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current"trainer/global_step2:JPGbdb_ DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated 866205078528 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed 841553293472 GDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 NCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated336729 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed 862863791104 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current250 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed 906456278528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 UEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 43592487424 \LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 43679917056 GDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11814 NCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed336727 ]MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 44920055808 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ODDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated343826 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak514 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 J?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed336727 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250522112 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated 841570332832 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 RADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed 906456278528 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11814 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 VEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated 885141475884 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed199245 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 K@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed343574 \LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 43571143052 % _timestamp1.7857953141076314e+09 YIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 43679917056 XHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 43592487424 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated 862880830464 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 C8DeviceStatsMonitor.on_train_batch_start/active.all.freed861378 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 trainer/global_step49 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11885 RGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated336729 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 RADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed 885038633400 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874880 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 _step1 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250522112 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 XHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 43485339928 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940454912 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated 862880830464 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated144579 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed 866180174848 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032c5fba >DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed"trainer/global_step2:JPkgbb ?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current"trainer/global_step2:JP 0rbm JDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak"trainer/global_step2:JPҍOagbb ?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak"trainer/global_step2:JP;u[bb] :DeviceStatsMonitor.on_train_batch_end/active.all.allocated"trainer/global_step2:JPY9]bX 5DeviceStatsMonitor.on_train_batch_end/active.all.peak"trainer/global_step2:JPymbh EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak"trainer/global_step2:JP^0db_ DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak"trainer/global_step2:JPsjbe BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated"trainer/global_step2:JPAqbl IDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated"trainer/global_step2:JP @,db_ DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated"trainer/global_step2:JPhbc @DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current"trainer/global_step2:JPnbi FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed"trainer/global_step2:JPqbl IDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated"trainer/global_step2:JP0sbn KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed"trainer/global_step2:JP[ibd ADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current"trainer/global_step2:JPDgbb ?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed"trainer/global_step2:JP׾-jbe BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated"trainer/global_step2:JPj$^bY 6DeviceStatsMonitor.on_train_batch_end/active.all.freed"trainer/global_step2:JP8bb] :DeviceStatsMonitor.on_train_batch_end/allocation.all.freed"trainer/global_step2:JPfKdhbc @DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak"trainer/global_step2:JP&kgbb ?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current"trainer/global_step2:JPeb` =DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed"trainer/global_step2:JP]rbm JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated"trainer/global_step2:JPzUkbf CDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak"trainer/global_step2:JPҜhbc @DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed"trainer/global_step2:JPhhbc @DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak"trainer/global_step2:JPlPlbg DDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated"trainer/global_step2:JPkbf CDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated"trainer/global_step2:JP.fba >DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed"trainer/global_step2:JP他wibd ADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated"trainer/global_step2:JPT&nbi FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed"trainer/global_step2:JPa)3mbh EDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed"trainer/global_step2:JPLHbb] :DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams"trainer/global_step2:JP`jbe BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated"trainer/global_step2:JPɴkbf CDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed"trainer/global_step2:JPb"pbk HDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current"trainer/global_step2:JPiEmbh EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak"trainer/global_step2:JP*;ibd ADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed"trainer/global_step2:JPl^bY 6DeviceStatsMonitor.on_train_batch_end/num_device_alloc"trainer/global_step2:JPDF/cb^ ;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak"trainer/global_step2:JP1 sbn KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed"trainer/global_step2:JP2eb` =DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed"trainer/global_step2:JP~Bpbk HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated"trainer/global_step2:JPolbg DDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed"trainer/global_step2:JP?>gbb ?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current"trainer/global_step2:JPt%eb` =DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed"trainer/global_step2:JPhbc @DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated"trainer/global_step2:JPX3fba >DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed"trainer/global_step2:JPYTwbr ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated"trainer/global_step2:JPíjbe BDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current"trainer/global_step2:JPknmbh EDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak"trainer/global_step2:JP'-}obj GDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current"trainer/global_step2:JPmbh EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed"trainer/global_step2:JP-ibd ADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated"trainer/global_step2:JP7.qbl IDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated"trainer/global_step2:JP /obj GDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current"trainer/global_step2:JPX3pbk HDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current"trainer/global_step2:JP cb^ ;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated"trainer/global_step2:JP-1/_bZ 7DeviceStatsMonitor.on_train_batch_end/segment.all.freed"trainer/global_step2:JP5dM^bY 6DeviceStatsMonitor.on_train_batch_end/segment.all.peak"trainer/global_step2:JP\ #gbb ?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed"trainer/global_step2:JPubp MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current"trainer/global_step2:JP<,ވ`b[ 8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections"trainer/global_step2:JP 4fba >DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak"trainer/global_step2:JPKpbk HDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current"trainer/global_step2:JPjpab\ 9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak"trainer/global_step2:JPȌmbh EDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated"trainer/global_step2:JPJlbg DDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak"trainer/global_step2:JPlbg DDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak"trainer/global_step2:JPrbm JDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak"trainer/global_step2:JPlubp MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current"trainer/global_step2:JP8{]bX 5DeviceStatsMonitor.on_train_batch_end/num_device_free"trainer/global_step2:JPm\bW 4DeviceStatsMonitor.on_train_batch_end/max_split_size"trainer/global_step2:JPU~chbc @DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current"trainer/global_step2:JP^lbg DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak"trainer/global_step2:JP.feb` =DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak"trainer/global_step2:JP5db_ DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current"trainer/global_step2:JPTVwbr ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated"trainer/global_step2:JP;db_ DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed"trainer/global_step2:JPrbm JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated"trainer/global_step2:JP+63Cpbk HDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current"trainer/global_step2:JPkrbm JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated"trainer/global_step2:JPzRCqbl IDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated"trainer/global_step2:JPJeb` =DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak"trainer/global_step2:JPa^2.) _wandbruntime27  _runtime27[L FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 _ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 45951886336 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated 858310505152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ZJDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 44448616386 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 E:DeviceStatsMonitor.on_train_batch_end/active.all.allocated888216 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 MDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated888216 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current257 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed 880030554112 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated203184 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed 883387291648 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 P?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed 902656278942 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 A6DeviceStatsMonitor.on_train_batch_end/active.all.freed878544 E:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed878544 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 ZJDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 44559543808 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 TCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated 902759121538 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 LADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated343372 VFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 44472107008 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed147176 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 SCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 44472107008 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed535174 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 [KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 45829607936 H=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed343370 YHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated 929364081664 UDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed 929216899584 _step2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 H=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed535174 Q@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated 924607137280 I>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed350358 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated 883412195328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KEDeviceStatsMonitor.on_train_batch_end/reEserved_bytes.large_pool.freed0 LADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated544844 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated147431 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current255 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 P?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed 924502661120 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 PEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated544844 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak504 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 122278400 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 \r&L J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 [KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 45829607936 H=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed343370 YHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated 929364081664 UDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed 929216899584 _step2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 H=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed535174 Q@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated 924607137280 I>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed350358 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated 883412195328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 LADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated544844 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated147431 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current255 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 P?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed 924502661120 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 PEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated544844 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak504 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 122278400 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 _ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 45951886336 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated 858310505152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ZJDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 44448616386 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 E:DeviceStatsMonitor.on_train_batch_end/active.all.allocated888216 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 MDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated888216 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current257 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed 880030554112 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated203184 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed 883387291648 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 P?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed 902656278942 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 A6DeviceStatsMonitor.on_train_batch_end/active.all.freed878544 E:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed878544 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 ZJDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 44559543808 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 TCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated 902759121538 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 LADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated343372 VFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 44472107008 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed147176 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 SCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 44472107008 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed535174 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251Ab< loss/FRt2/assignment_loss"trainer/global_step2:JPWFCb> loss/SRqqt2/assignment_loss"trainer/global_step2:JPc#Bb= loss/SRqqt1/detection_loss"trainer/global_step2:JPBb= loss/SRqqt2/detection_loss"trainer/global_step2:JPP OAb< loss/FRt1/assignment_loss"trainer/global_step2:JP)Cb> loss/SRqqt1/assignment_loss"trainer/global_step2:JP9+Ab< loss/FBt1/assignment_loss"trainer/global_step2:JP<)Ab< loss/FBt2/assignment_loss"trainer/global_step2:JP@nߠ@b; loss/FRt1/detection_loss"trainer/global_step2:JPg@b; loss/FBt1/detection_loss"trainer/global_step2:JP0-b( epoch"trainer/global_step2:JPqb@b; loss/FBt2/detection_loss"trainer/global_step2:JP/*P@b; loss/FRt2/detection_loss"trainer/global_step2:JP7b2 loss/total_loss"trainer/global_step2:JPQ .) _wandbruntime27  _runtime27}: .loss/FBt2/detection_loss22.13230026644819 _step3 .loss/FRt2/detection_loss3.242439370740343 #loss/total_loss74.390704210884 /loss/FRt2/assignment_loss5.798686552930762 1loss/SRqqt2/assignment_loss4.551258597575443 0loss/SRqqt1/detection_loss3.255451018659814 0loss/SRqqt2/detection_loss4.977281785347093 trainer/global_step49 _runtime 27.305308136 0loss/FRt1/assignment_loss5.6610496010578855 1loss/SRqqt1/assignment_loss4.532234724285533 0loss/FBt1/assignment_loss2.4131935901112027 0loss/FBt2/assignment_loss2.3265245381523583 .loss/FRt1/detection_loss4.751960010595725 /loss/FBt1/detection_loss10.748324154979652 epoch0 % _timestamp1.7857953144824657e+09 _step3 .loss/FRt2/detection_loss3.242439370740343 #loss/total_loss74.390704210884 /loss/FRt2/assignment_loss5.798686552930762 1loss/SRqqt2/assignment_loss4.551258597575443 0loss/SRqqt1/detection_loss3.255451018659814 0loss/SRqqt2/detection_loss4.977281785347093 trainer/global_step49 _runtime 27.305308136 0loss/FRt1/assignment_loss5.6610496010578855 1loss/SRqqt1/assignment_loss4.532234724285533 0loss/FBt1/assignment_loss2.4131935901112027 0loss/FBt2/assignment_loss2.3265245381523583 .loss/FRt1/detection_loss4.751960010595725 /loss/FBt1/detection_loss10.748324154979652 epoch0 % _timestamp1.7857953144824657e+09 .loss/FBt2/detection_loss22.13230026644819G(: ɔU gpu.0.gpu19.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp33.0 gpu.0.powerWatts124.226' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent41.40866666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0̞!(:  ʇU) proc.memory.availableMB 38528.6953125 disk.drbd1550.out0 cpu18.121416901502574 disk.drbd1289.out0 disk.dm-0.out 1.7734375 disk.drbd1673.in0 disk.drbd1226.in0 disk.drbd2078.out0 disk.drbd1965.out0# disk./.usageGB578.329891204834 disk.drbd1289.in0 disk.sda.in0 disk.sda1.out0 disk.drbd1510.in0 disk.nvme0n1.out 1.7734375 disk.drbd1356.out0 disk.drbd2111.out0) proc.memory.percent17.18199849128723* disk./.usagePercent32.343235045119755 disk.drbd2041.out0 disk.dm-0.in0 network.recv43481 disk.drbd1552.out0 disk.drbd1828.in0 disk.sda.out1.328125 disk.drbd1285.in0 disk.drbd1285.out0 disk.sda2.out1.328125 disk.drbd1584.in0 disk.drbd1828.out0 disk.drbd1510.out0 disk.drbd1226.out0 disk.drbd2078.in0 disk.drbd1552.in0 disk.rbd1.in0" memory_percent41.209876537323 disk.drbd1938.in0 disk.rbd0.out0 disk.drbd1584.out0 disk.drbd2111.in0 disk.drbd1817.in0 disk.drbd1965.in0 disk.drbd2041.in0 disk.rbd0.in0 disk.drbd2118.in0 disk.nvme0n1.in0 disk.drbd1931.in0 network.sent50332 disk.rbd1.out0 disk.drbd1673.out0 disk.drbd1931.out0 proc.cpu.threads29 disk.drbd1550.in0 disk.drbd1938.out0 disk.sda1.in0$ proc.memory.rssMB11260.39453125 disk.drbd2118.out0 disk.drbd1356.in0 disk.sda2.in0 disk.drbd1817.out0 q(: U gpu.0.gpu37.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp34.0 gpu.0.powerWatts101.182' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent33.727333333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0ȓ %(:  U disk.drbd1356.out0 disk.drbd1817.out0$ memory_percent41.20981693267822 cpu17.572258702850956 proc.cpu.threads29 disk.dm-0.in0 disk.drbd1584.out0 disk.drbd1226.out0 disk.drbd2118.in0" disk.nvme0n1.out 2.19873046875 disk.rbd1.in0 disk.drbd1673.in0 disk.drbd2041.in0 disk.drbd1552.in0 disk.drbd1938.in0 disk.drbd1828.in0 disk.sda.out1.5625 disk.drbd2078.in0 disk.drbd1584.in0 disk.drbd1510.in0 disk.drbd1510.out0 disk.sda1.in0 disk.drbd2111.in0 disk.drbd1931.in0 disk.sda2.in0 disk.drbd1289.out0 disk.drbd1828.out0 disk.sda2.out1.5625 network.recv47212) disk./.usagePercent32.34323824518737 disk.drbd2118.out0# proc.memory.rssMB 11260.3984375 disk.drbd2111.out0 disk.drbd1673.out0 disk.drbd1931.out0( proc.memory.availableMB 38528.734375) proc.memory.percent17.18200445175171 disk.drbd1965.in0 disk.drbd2041.out0 disk.drbd2078.out0 network.sent101006# disk./.usageGB578.329948425293 disk.rbd1.out0 disk.drbd1226.in0 disk.drbd1550.in0 disk.drbd1285.in0 disk.drbd1356.in0 disk.drbd1817.in0 disk.drbd1552.out0 disk.drbd1289.in0 disk.nvme0n1.in0 disk.drbd1965.out0 disk.drbd1550.out0 disk.drbd1938.out0 disk.sda.in0 disk.dm-0.out 2.19873046875 disk.sda1.out0 disk.drbd1285.out0 disk.rbd0.in0 disk.rbd0.out0W.) _wandbruntime50  _runtime50F! % _timestamp1.7857953374794846e+09 _runtime 50.302472718 _step4 ' lr-AdamW/pg12.5514464620453036e-06 ' lr-AdamW/pg22.5514464620453036e-06 trainer/global_step99 A b ' lr-AdamW/pg22.5514464620453036e-06 trainer/global_step99 % _timestamp1.7857953374794846e+09 _runtime 50.302472718 _step4 ' lr-AdamW/pg12.5514464620453036e-06'ԅ^.) _wandbruntime50  _runtime50Dٵ|'N C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed 1808790564352 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated 1721218980864 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 RGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated668879 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed 1726536014848 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak522 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 _step5 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 87588622848 K@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed683141 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ODDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated683400 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak512 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed 1678561909472 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated1731188 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed 1808790564352 \LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 87676052480 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated 1766038555392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 NCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated668879 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 \LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 87459606560 _runtime 50.303748473 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed1721530 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated 1808895033344 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated 1816811995648 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed 1816652223488 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 XHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 87373803436 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed 1721201941504 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated 1678578948832 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 ]MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 90116208640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed287046 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 trainer/global_step99 HDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed1052653 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 J?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed668877 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed1052653 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 NCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed668877 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated1062309 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current259 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated287303 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated396097 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated 1808895033344 HDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed1052653 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 J?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed668877 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed1052653 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 NCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed668877 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated1062309 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current259 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated287303 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated396097 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated 1808895033344 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated 1721218980864 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 RGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated668879 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed 1726536014848 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak522 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 _step5 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 87588622848 K@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed683141 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ODDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated683400 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak512 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed 1678561909472 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated1731188 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed 1808790564352 \LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 87676052480 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated 1766038555392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 NCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated668879 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 \LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 87459606560 _runtime 50.303748473 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed1721530 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated 1808895033344 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated 1816811995648 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed 1816652223488 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 XHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 87373803436 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed 1721201941504 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated 1678578948832 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 ]MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 90116208640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed287046 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 trainer/global_step99.) _wandbruntime50  _runtime50:lc&L JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated1072846 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed1738696 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed1063176 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated 1826941423104 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated 1783656201046 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed 1783553358450 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 LADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed675520 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated290166 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 SCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 88468242432 ZJDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 88337079894 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 NDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed689957 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated400034 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated 1834919367680 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed 1743743131648 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak522 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated 1743768035328 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed289925 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated 1826941423104 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current243 _ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 91151332352 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 _step6 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated1748368 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 VFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 88468242432 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated 1738385743872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 trainer/global_step99 % _timestamp1.7857953378577547e+09 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed400032 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed1063176 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed 1834761699840 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated 1738385743872 WGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 88555679232 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated1748368 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated1072846 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed 1826836946944 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed 1695302081792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 H=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed675520 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current241 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed 1738368704512B r&M [KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 91018568192 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated 1695319121152 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 LADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated675522 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 _runtime 50.68109903 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 _step6 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated1748368 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 VFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 88468242432 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated 1738385743872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 trainer/global_step99 % _timestamp1.7857953378577547e+09 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed400032 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed1063176 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed 1834761699840 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated 1738385743872 WGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 88555679232 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated1748368 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated1072846 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed 1826836946944 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed 1695302081792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 H=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed675520 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current241 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed 1738368704512 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated1072846 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed1738696 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed1063176 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated 1826941423104 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated 1783656201046 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed 1783553358450 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 LADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed675520 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated290166 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 SCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 88468242432 ZJDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 88337079894 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 NDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed689957 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated400034 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated 1834919367680 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed 1743743131648 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak522 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated 1743768035328 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed289925 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated 1826941423104 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current243 _ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 91151332352 EDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed 2646832792416 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed430122 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated 2711229319168 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated1590311 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated592947 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated1001029 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak522 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed2581682 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed 2711124850176 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008i'O HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed 2711124850176 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated 2722388068352 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed 2646832792416 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed430122 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated 2711229319168 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated1590311 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated592947 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated1001029 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak522 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed2581682 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 .) _wandbruntime73  _runtime73Qe&M YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated 2596723894272 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated 2596723894272 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed1591178 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed1029887 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed 2740344391680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed 2729171232768 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated2608520 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated1600848 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak512 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated 2740495768064 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 136371892736 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated 2729275708928 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 132139740166 % _timestamp1.7857953606644635e+09 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 132551814656 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed2598848 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 132551814656 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated1007672 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed1007670 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 126472704 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated1600848 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed1591178 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 136245420032 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated 2729275708928 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 151376384 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 132225543402 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed 2596706854912 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed 2596706854912 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed 2664450437958 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 132464377856 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated1030145 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed2598848 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed596882 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed 2532310697792 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 _step10 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current256 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated1007672 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated 2532327737152 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current258 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 trainer/global_step149JVM Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed 2729171232768 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated2608520 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated1600848 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak512 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated 2740495768064 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 136371892736 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated 2729275708928 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 132139740166 % _timestamp1.7857953606644635e+09 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 132551814656 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed2598848 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 132551814656 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated1007672 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed1007670 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 126472704 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated1600848 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed1591178 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 136245420032 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated 2729275708928 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 151376384 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 132225543402 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed WQ;D2596706854912 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed 2596706854912 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed 2664450437958 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 132464377856 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated1030145 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed2598848 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed596882 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed 2532310697792 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 _step10 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current256 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated1007672 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated 2532327737152 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current258 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 trainer/global_step149 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated 2596723894272 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated 2596723894272 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed1591178 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed1029887 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed 2740344391680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 | .) _wandbruntime73  _runtime73=G 1loss/SRqqt1/detection_loss2.7099583333029464 trainer/global_step149 $ _timestamp1.785795360666424e+09 _runtime 73.48947321 2loss/SRqqt1/assignment_loss4.5277445656912665 /loss/FRt1/detection_loss3.5964532215749063 .loss/FRt2/detection_loss3.514197044682746 .loss/FBt1/detection_loss14.85692478945622 %loss/total_loss70.04734856533052 0loss/SRqqt2/detection_loss5.569589744892813 /loss/FBt2/detection_loss14.634355749487877 /loss/FRt1/assignment_loss5.690389043541365 /loss/FRt2/assignment_loss5.780202573659468 0loss/FBt1/assignment_loss2.3890942335128784 epoch0 _step11 1loss/SRqqt2/assignment_loss4.467312489786456 /loss/FBt2/assignment_loss2.311126775741577?Q .loss/FBt1/detection_loss14.85692478945622 %loss/total_loss70.04734856533052 0loss/SRqqt2/detection_loss5.569589744892813 /loss/FBt2/detection_loss14.634355749487877 /loss/FRt1/assignment_loss5.690389043541365 /loss/FRt2/assignment_loss5.780202573659468 0loss/FBt1/assignment_loss2.3890942335128784 epoch0 _step11 1loss/SRqqt2/assignment_loss4.467312489786456 /loss/FBt2/assignment_loss2.311126775741577 1loss/SRqqt1/detection_loss2.7099583333029464 trainer/global_step149 $ _timestamp1.785795360666424e+09 _runtime 73.48947321 2loss/SRqqt1/assignment_loss4.5277445656912665 /loss/FRt1/detection_loss3.5964532215749063 .loss/FRt2/detection_loss3.514197044682746 B8(: ᏫU gpu.0.gpu23.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp35.0 gpu.0.powerWatts133.326' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent44.442 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0" 0(:  U disk.drbd1817.out0 disk.drbd1931.in0 disk.sda.in0 disk.drbd1584.in0 disk.nvme0n1.in0 disk.drbd2111.in0 disk.sda.out 2.33203125 disk.drbd2118.out0 disk.drbd1289.out0 disk.drbd1510.out0 disk.drbd1226.in0 disk.drbd2118.in0 disk.rbd1.in0 network.sent149331 disk.sda2.in0 disk.drbd1550.out0 disk.rbd0.out0 disk.dm-0.in0 disk.drbd1226.out0" disk.nvme0n1.out 3.63916015625 disk.drbd1356.out0 cpu16.810125642255038 disk.drbd1285.out0* proc.memory.availableMB38526.65234375 disk.drbd1289.in0 disk.drbd2111.out0 disk.drbd1552.out0 disk.drbd1938.out0 disk.sda1.in0 disk.dm-0.out 3.63916015625 disk.drbd1510.in0 disk.drbd1931.out0 disk.drbd2041.in0 disk.drbd1938.in0 disk.rbd0.in0 disk.drbd1584.out0 disk.drbd1828.in0 disk.drbd1965.in0 disk.drbd2041.out0# proc.memory.rssMB 11260.3984375 disk.rbd1.out0 disk.drbd1965.out0 disk.drbd1673.out0 disk.drbd1550.in0 disk.drbd2078.in0 proc.cpu.threads29) disk./.usagePercent32.34324336529556 disk.drbd1817.in0 disk.drbd2078.out0 disk.drbd1285.in0 disk.drbd1828.out0 network.recv51895 disk.sda2.out 2.33203125 disk.drbd1552.in0 disk.sda1.out0$ disk./.usageGB578.3300399780273) proc.memory.percent17.18200445175171 disk.drbd1673.in0 disk.drbd1356.in0$ memory_percent41.21299386024475=(: U gpu.0.gpu41.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp36.0 gpu.0.powerWatts98.97' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent32.989999999999995 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0dH)(:  U disk.dm-0.out 4.17041015625 disk.drbd1510.out0 network.sent194092 disk.drbd1356.out0 disk.drbd2041.in0" disk.nvme0n1.out 4.17041015625 disk.drbd1673.in0 disk.drbd1552.out0 disk.sda1.out0 disk.drbd1285.in0 disk.sda2.in0 disk.drbd2111.out0 disk.rbd1.out0) proc.memory.availableMB 38526.7890625 network.recv54205 disk.drbd1285.out0 disk.drbd1817.out0 disk.drbd1965.out0 proc.cpu.threads29 disk.drbd1510.in0 disk.drbd1289.in0 disk.drbd1938.in0 disk.drbd1938.out0 disk.drbd2078.out0 disk.drbd1552.in0 disk.drbd1828.out0 disk.drbd1550.in0 disk.nvme0n1.in0 disk.drbd1931.in0! proc.memory.rssMB 11260.46875 disk.drbd1226.in0 disk.drbd2118.in0 disk.sda2.out 2.64453125 disk.drbd1289.out0 disk.rbd0.out0 disk.sda.in0 disk.dm-0.in0 disk.drbd1356.in0 disk.drbd1828.in0 disk.rbd0.in0 disk.sda.out 2.64453125* proc.memory.percent17.182111740112305) disk./.usagePercent32.34324549867396 disk.drbd1817.in0 disk.drbd1226.out0 disk.drbd1584.in0 disk.drbd1584.out0 disk.drbd2111.in0 disk.sda1.in0 disk.drbd1931.out0$ memory_percent41.21278524398804 disk.drbd1550.out0 disk.drbd2118.out0 disk.drbd1965.in0 cpu16.54950307910788 disk.drbd2041.out0 disk.drbd2078.in0 disk.rbd1.in0 disk.drbd1673.out0 disk./.usageGB 578.330078125.) _wandbruntime95  _runtime95A & lr-AdamW/pg25.128665110575913e-06 trainer/global_step199 % _timestamp1.7857953828023703e+09 _runtime 95.625337196 _step12 & lr-AdamW/pg15.128665110575913e-06.w & lr-AdamW/pg15.128665110575913e-06 & lr-AdamW/pg25.128665110575913e-06 trainer/global_step199 % _timestamp1.7857953828023703e+09 _runtime 95.625337196 _step12 Y.) _wandbruntime95  _runtime95sm0'O C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated1333179 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed1333177 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed 3627730790912 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed1333177 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed 3613459136000 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated3451492 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed2108657 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated 3352596180832 _step13 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed1363117 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed 3527729871924 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated1333179 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated 3527832714408 _runtime 95.626659608 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed 3613459136000 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed3441834 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 180617964544 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 180483096064 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated1363379 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated789797 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 175668323328 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 HDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed 3613459136000 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated3451492 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed2108657 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated 3352596180832 _step13 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed1363117 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed 3527729871924 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated1333179 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated 3527832714408 _runtime 95.626659608 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed 3613459136000 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed3441834 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 180617964544 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 180483096064 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated1363379 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated789797 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 175668323328 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 HDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated1339822 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed 3464454811648 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 181515510272 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 176114006910 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 QBDeviceStatsMonitor.on_train_batch_end/rese rved_bytes.all.allocated 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 trainer/global_step199 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed793732 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed1369888 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed 3645845946368 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 % _timestamp1.7857953831862667e+09 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated 3631609994752 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated 3464479715328 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed3459000 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated3468672 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 176460513280 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated 3631609994752 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 149279232 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak516 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed1339820 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated 3455062044672 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated576397 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated 3545450360062 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed 3455045005312 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 176547950080 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 _step14 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 176547950080 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed 3455045005312 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed2119180 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated2128850 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed3459000 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 124375552 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0U&M M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated3468672 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 176460513280 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated 3631609994752 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 149279232 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak516 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed1339820 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated 3455062044672 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated576397 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated 3545450360062 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed 3455045005312 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 176547950080 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 _step14 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 176547950080 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed 3455045005312 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed2119180 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated2128850 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed3459000 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 124375552 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated1339822 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed 3545347517466 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated1339822 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed 3464454811648 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 181515510272 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 176114006910 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 trainer/global_step199 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed793732 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed1369888 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed 3645845946368 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 % _timestamp1.7857953831862667e+09 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated 3631609994752 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated 3464479715328 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed3459000 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160B.) _wandbruntime96  _runtime96T 1loss/SRqqt2/assignment_loss4.578759358042762 .loss/FBt2/detection_loss19.19500277152187 %loss/total_loss78.93283339446624 % _timestamp1.7857953831880267e+09 /loss/FRt1/assignment_loss5.672726470070916 1loss/SRqqt1/assignment_loss4.535656733555837 0loss/FBt1/assignment_loss2.3531232357025145 .loss/FRt1/detection_loss4.492855467506357 -loss/FRt2/detection_loss3.62628174519416 0loss/SRqqt1/detection_loss2.928328666198361 /loss/FBt1/detection_loss19.333837734162806 trainer/global_step199 0loss/FBt2/assignment_loss2.2959651696054557 1loss/SRqqt2/detection_loss4.0683959520288875 epoch0 _runtime 96.010867016 /loss/FRt2/assignment_loss5.851900090876314 _step15 /loss/FRt1/assignment_loss5.672726470070916 1loss/SRqqt1/assignment_loss4.535656733555837 0loss/FBt1/assignment_loss2.3531232357025145 .loss/FRt1/detection_loss4.492855467506357 -loss/FRt2/detection_loss3.62628174519416 0loss/SRqqt1/detection_loss2.928328666198361 /loss/FBt1/detection_loss19.333837734162806 trainer/global_step199 0loss/FBt2/assignment_loss2.2959651696054557 1loss/SRqqt2/detection_loss4.0683959520288875 epoch0 _runtime 96.010867016 /loss/FRt2/assignment_loss5.851900090876314 _step15 1loss/SRqqt2/assignment_loss4.578759358042762 .loss/FBt2/detection_loss19.19500277152187 %loss/total_loss78.93283339446624 % _timestamp1.7857953831880267e+09d(: ÷U gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp36.0 gpu.0.powerWatts128.659' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.88633333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0U?,(:  U disk.rbd0.out0 disk.drbd1226.in0 disk.drbd1584.in0 disk.drbd1673.in0 disk.drbd1938.out0 disk.drbd2078.out0 disk.drbd1817.out0 disk.drbd1552.in0 disk.sda2.out 3.04296875 disk.drbd2111.in0 disk.drbd1550.out0 disk.drbd1289.in0 disk.nvme0n1.in0 disk.rbd1.out0 disk.drbd1552.out0 disk.drbd1938.in0 disk.sda.in0 proc.cpu.threads29( proc.memory.availableMB 38526.296875 disk.drbd2118.in0" disk.nvme0n1.out 4.36181640625 disk.drbd2111.out0 disk.drbd1931.out0 cpu16.32930401193619 disk.drbd1356.in0 disk.drbd1828.out0* proc.memory.percent17.182111740112305 disk.drbd1510.out0 disk.dm-0.in0 disk.sda2.in0 network.recv55688) disk./.usagePercent32.34324656536317 disk.drbd2118.out0 disk.drbd1356.out0 disk.sda1.in0 disk.drbd1510.in0 disk.dm-0.out 4.36181640625 disk.drbd1931.in0 disk.drbd1584.out0 disk.drbd1965.in0 disk.drbd1285.in0 disk.drbd1285.out0 disk.rbd1.in0 disk.drbd2041.out0 disk.drbd1289.out0 disk.drbd1550.in0 disk.rbd0.in0 disk.sda.out 3.04296875 disk.sda1.out0$ memory_percent41.21353626251221 disk.drbd1226.out0 disk.drbd2078.in0 disk.drbd1817.in0 network.sent197431$ disk./.usageGB578.3300971984863 disk.drbd1828.in0 disk.drbd2041.in0! proc.memory.rssMB 11260.46875 disk.drbd1965.out0 disk.drbd1673.out0؋"0+ _wandbruntime118  _runtime118m trainer/global_step249 % _timestamp1.7857954053321607e+09 _runtime 118.155140932 _step16 & lr-AdamW/pg16.417274434841219e-06 & lr-AdamW/pg26.417274434841219e-06 & lr-AdamW/pg16.417274434841219e-06 & lr-AdamW/pg26.417274434841219e-06 trainer/global_step249 % _timestamp1.7857954053321607e+09 _runtime 118.155140932 _step16M0+ _wandbruntime118  _runtime118z_W'O VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed 4515793421824 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed986645 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated2646315 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak527 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 225762959360 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated2646315 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 219124997084 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed 4296216392704 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated4311644 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated 4515897890816 HDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed 4515793421824 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed 4296216392704 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 219664458752 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated 4408729793916 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated 4189604796832 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 219039193960 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated 4296233432064 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed1703268 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _step17 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 219577029120 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 219664458752 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed716623 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed2636659 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current242 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated 4533391397888 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed 4189587757472 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 % _timestamp1.7857954053331704e+09 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 _runtime 118.156507363 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated1665329 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated986647 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed 4307603534848 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 trainer/global_step249 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed2636659 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed4301986 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed1665327 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated716863 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated1703510 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2ZO L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated4311644 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated 4515897890816 HDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed 4515793421824 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed 4296216392704 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 219664458752 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated 4408729793916 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated 4189604796832 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 219039193960 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated 4296233432064 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed1703268 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _step17 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 219577029120 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 219664458752 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed716623 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed2636659 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current242 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated 4533391397888 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed 4189587757472 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 % _timestamp1.7857954053331704e+09 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 _runtime 118.156507363 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated1665329 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated986647 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed 4307603534848 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 trainer/global_step249 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed2636659 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed4301986 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed1665327 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated716863 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated1703510 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed 4515793421824 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed986645 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated2646315 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak527 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 225762959360 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated2646315 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 219124997084 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed 42962163927040+ _wandbruntime118  _runtime118(&M MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed990582 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 trainer/global_step249 _step18 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated 4426347439570 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed2647182 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated 4313400195072 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated 4313400195072 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 220002470418 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 220456648704 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated719708 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed 4313383155712 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current245 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated4328824 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed 4324810651648 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated1671972 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 % _timestamp1.7857954057100792e+09 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated2656852 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 219916667182 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed1710047 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 226530671616 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated1671972 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated4328824 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated1710292 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak527 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed 4313383155712 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed719465 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _runtime 118.533439089 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated 4533944280576 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 220544085504 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed1671970 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current243 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated 4206344969152 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated 4324835555328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed4319152 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed 4206327929792 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed1671970 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed4319152 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated990584 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed 4551341323264 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320Ɩ&M TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed990582 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 trainer/global_step249 _step18 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated 4426347439570 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed2647182 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated 4313400195072 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated 4313400195072 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 220002470418 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 220456648704 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated719708 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed 4313383155712 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current245 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated4328824 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed 4324810651648 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated1671972 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 % _timestamp1.7857954057100792e+09 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated2656852 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 219916667182 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed1710047 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 226530671616 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated1671972 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated4328824 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated1710292 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak527 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed 4313383155712 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed719465 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _runtime 118.533439089 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated 4533944280576 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 220544085504 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed1671970 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current243 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated 4206344969152 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated 4324835555328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed4319152 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed 4206327929792 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed1671970 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed4319152 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated990584 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed 4551341323264 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0FR0+ _wandbruntime118  _runtime118& 1loss/SRqqt2/assignment_loss4.508640909194947 epoch0 /loss/FRt1/assignment_loss5.676941976314637 0loss/FBt1/assignment_loss2.2763679790496827 .loss/FRt2/detection_loss3.765530929770521 0loss/SRqqt1/detection_loss3.120020568512735 /loss/FBt2/detection_loss17.894018699228763 trainer/global_step249 _runtime 118.534745748 .loss/FRt1/detection_loss4.057483811204026 /loss/FBt1/detection_loss15.151400722265244 % _timestamp1.7857954057119086e+09 %loss/total_loss72.49814690597984 1loss/SRqqt1/assignment_loss4.493164968490601 0loss/FBt2/assignment_loss2.2594335079193115 0loss/SRqqt2/detection_loss3.490356454410051 _step19 /loss/FRt2/assignment_loss5.804786379619311U /loss/FRt2/assignment_loss5.804786379619311 1loss/SRqqt2/assignment_loss4.508640909194947 epoch0 /loss/FRt1/assignment_loss5.676941976314637 0loss/FBt1/assignment_loss2.2763679790496827 .loss/FRt2/detection_loss3.765530929770521 0loss/SRqqt1/detection_loss3.120020568512735 /loss/FBt2/detection_loss17.894018699228763 trainer/global_step249 _runtime 118.534745748 .loss/FRt1/detection_loss4.057483811204026 /loss/FBt1/detection_loss15.151400722265244 % _timestamp1.7857954057119086e+09 %loss/total_loss72.49814690597984 1loss/SRqqt1/assignment_loss4.493164968490601 0loss/FBt2/assignment_loss2.2594335079193115 0loss/SRqqt2/detection_loss3.490356454410051 _step19%(: ϦU gpu.0.gpu24.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp37.0 gpu.0.powerWatts122.589' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent40.863 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0Ïa (:  ϦU network.recv58559 disk.sda.in0 disk.sda1.out0 disk.drbd1226.out0 disk.drbd2111.out0 disk.nvme0n1.in0 disk.drbd1356.in0 disk.drbd1356.out0 disk.drbd1817.in0 disk.drbd2041.in0* proc.memory.percent17.182111740112305 disk.dm-0.out 4.7333984375$ memory_percent41.21348261833191 cpu16.13522353058109 disk.sda.out 3.43359375 disk.sda2.in0$ disk./.usageGB578.3300590515137 disk.drbd1584.in0 disk.rbd1.in0* proc.memory.availableMB38526.33203125 disk.rbd0.out0 disk.drbd1285.out0 disk.drbd1289.in0 disk.dm-0.in0 disk.drbd1552.in0 disk.drbd1550.out0 disk.drbd1285.in0 disk.drbd2078.out0 disk.drbd2111.in0 disk.drbd1931.in0 disk.drbd1226.in0 disk.drbd1673.out0 proc.cpu.threads29 disk.drbd1828.in0 disk.drbd1510.in0 disk.drbd1510.out0 disk.rbd1.out0 disk.drbd1965.in0 disk.drbd1673.in0 network.sent242442 disk.rbd0.in0 disk.drbd1931.out0 disk.drbd1552.out0! proc.memory.rssMB 11260.46875 disk.drbd1817.out0 \disk.drbd2041.out0) disk./.usagePercent32.34324443198476 disk.drbd1828.out0 disk.drbd1289.out0 disk.drbd2078.in0 disk.drbd2118.out0 disk.sda1.in0 disk.drbd1584.out0 disk.sda2.out 3.43359375 disk.drbd1938.in0 disk.drbd1938.out0 disk.drbd1965.out0 disk.drbd1550.in0 disk.drbd2118.in0! disk.nvme0n1.out 4.73339843755 (: ަU gpu.0.gpu29.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp37.0 gpu.0.powerWatts125.635' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent41.87833333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0?-(:  ަV disk.sda2.in0 disk.drbd1289.out0 disk.sda.in0 disk.sda1.in0 disk.drbd1226.in0 disk.drbd1931.out0 disk.drbd1289.in0 network.recv62486 disk.drbd1965.in0 disk.drbd2041.in0$ disk./.usageGB578.3300018310547 disk.drbd2118.in0 disk.drbd2118.out0 disk.drbd1817.in0 disk.drbd1550.in0 disk.drbd1584.out0! disk.nvme0n1.out 4.9169921875 disk.drbd2111.out0 disk.drbd1938.in0* proc.memory.availableMB38525.91015625 disk.nvme0n1.in0 disk.drbd1552.in0 disk.drbd1817.out0 disk.drbd1828.out0 disk.sda1.out0 disk.drbd1285.out0 disk.drbd2078.out0 disk.drbd2111.in0 disk.drbd1965.out0 disk.drbd1673.in0 disk.drbd1673.out0 disk.rbd0.out0 disk.drbd2078.in0) disk./.usagePercent32.34324123191714 disk.rbd1.out0 disk.drbd1550.out0 disk.drbd1938.out0 disk.drbd1510.out0* proc.memory.percent17.182111740112305 disk.drbd1356.in0 disk.drbd1356.out0 disk.drbd1931.in0 disk.rbd0.in0 disk.drbd1510.in0! proc.memory.rssMB 11260.46875 proc.cpu.threads29 disk.dm-0.in0 cpu15.942684367671797 disk.sda.out 4.13671875 disk.drbd1285.in0 disk.dm-0.out 4.9169921875 disk.drbd1552.out0 disk.drbd1828.in0 disk.drbd1584.in0 disk.rbd1.in0 disk.drbd2041.out0 disk.drbd1226.out0$ memory_percent41.21412634849548 network.sent287733 disk.sda2.out 4.13671875 ˕0+ _wandbruntime140  _runtime140KS & lr-AdamW/pg27.705883759106523e-06 trainer/global_step299 % _timestamp1.7857954280162685e+09 _runtime 140.839226813 _step20 & lr-AdamW/pg17.705883759106523e-06b' & lr-AdamW/pg27.705883759106523e-06 trainer/global_step299 % _timestamp1.7857954280162685e+09 _runtime 140.839226813 _step20 & lr-AdamW/pg17.705883759106523e-06 0+ _wandbruntime140  _runtime140C'O ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 270799512064 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current224 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated 5167984278528 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated 5289626873424 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed 5418127707648 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed 5154554543104 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated3174317 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed3164661 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated860748 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed5162138 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed1997477 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed1183495 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed 5438758886912 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed3164661 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed2044019 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 % _timestamp1.7857954280172246e+09 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 270930186240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated1183497 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated1997479 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed 5418127707648 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated5171796 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed1997477 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 _step21 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated 5154571582464 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed 5289524030940 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 trainer/global_step299 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated 5418232176640 _runtime 140.840495509 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated 5418232176640 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated 5154571582464 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated2044245 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak538 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 263660594176 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed 5026596373472 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated3174317 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 263573164544 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed 5167959374848 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current224 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated 5167984278528 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated 5289626873424 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed 5418127707648 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed 5154554543104 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated3174317 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed3164661 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated860748 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed5162138 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed1997477 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed1183495 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed 5438758886912 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed3164661 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed2044019 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 % _timestamp1.7857954280172246e+09 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 270930186240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated1183497 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated1997479 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed 5418127707648 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated5171796 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed1997477 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 _step21 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated 5154571582464 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed 5289524030940 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 trainer/global_step299 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated 5418232176640 _runtime 140.840495509 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated 5418232176640 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated 5154571582464 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated2044245 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak538 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 263660594176 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed 5026596373472 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated3174317 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 25052313600+ _wandbruntime141  _runtime141(w!%M MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed3175184 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 _runtime 141.215832482 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated5188976 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak538 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current253 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed 5171721306112 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed2050790 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 _step22 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 264540220928 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed5179304 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 271844564480 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 271720188928 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated 5171738345472 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 124375552 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated 5457035959808 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed 5185166491648 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed 5043336545792 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed5179304 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated5188976 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated2004122 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 264540220928 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated 5307244519078 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current255 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 263890933926 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed2004120 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 264452784128 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed 5436174090240 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated1187434 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 % _timestamp1.7857954283924348e+09 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 149279232 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 264452784128 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated3184854 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated 5171738345472 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated3184854 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated 5436278566400 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated863611 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed 5307141676482 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated 5436278566400 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak528 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated 5185191395328 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated 5043353585152 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed2004120 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 263805130690 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 NDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed863358 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_tTyxrain_batch_end/requested_bytes.large_pool.current17039360 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 264452784128 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated3184854 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated 5171738345472 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated3184854 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated 5436278566400 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated863611 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed 5307141676482 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated 5436278566400 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak528 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated 5185191395328 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated 5043353585152 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed2004120 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 263805130690 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 NDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed863358 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 _runtime 141.215832482 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated5188976 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak538 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current253 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed 5171721306112 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed2050790 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 _step22 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 264540220928 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed5179304 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 271844564480 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 271720188928 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated 5171738345472 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 124375552 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated 5457035959808 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed 5185166491648 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed 5043336545792 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated1210+ _wandbruntime141  _runtime141瘮 %loss/total_loss87.58256426999878 trainer/global_step299 % _timestamp1.7857954283943334e+09 _step23 1loss/SRqqt1/assignment_loss4.543450847006681 -loss/FRt1/detection_loss4.25332493468737 epoch0 0loss/FRt1/assignment_loss5.6731762152451735 /loss/FRt2/assignment_loss5.929975618634905 1loss/SRqqt2/assignment_loss4.544027718375711 /loss/FBt1/assignment_loss2.249137850368724 .loss/FRt2/detection_loss3.239768941629501 1loss/SRqqt2/detection_loss3.8363463487695246 /loss/FBt1/detection_loss22.341675782028368 _runtime 141.21723711 0loss/FBt2/assignment_loss2.2075639196804593 1loss/SRqqt1/detection_loss2.7649189127903235 /loss/FBt2/detection_loss25.999197180782044 .loss/FRt2/detection_loss3.239768941629501 1loss/SRqqt2/detection_loss3.8363463487695246 /loss/FBt1/detection_loss22.341675782028368 _runtime 141.21723711 0loss/FBt2/assignment_loss2.2075639196804593 1loss/SRqqt1/detection_loss2.7649189127903235 /loss/FBt2/detection_loss25.999197180782044 %loss/total_loss87.58256426999878 trainer/global_step299 % _timestamp1.7857954283943334e+09 _step23 1loss/SRqqt1/assignment_loss4.543450847006681 -loss/FRt1/detection_loss4.25332493468737 epoch0 0loss/FRt1/assignment_loss5.6731762152451735 /loss/FRt2/assignment_loss5.929975618634905 1loss/SRqqt2/assignment_loss4.544027718375711 /loss/FBt1/assignment_loss2.249137850368724+Ȝ(: U gpu.0.gpu37.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp37.0 gpu.0.powerWatts117.024' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent39.007999999999996 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0F/(:  V$ memory_percent41.21459126472473 disk.drbd1931.out0 disk.sda1.out0 disk.drbd1285.out0$ disk./.usageGB578.3303031921387 disk.drbd1289.in0 disk.drbd1285.in0 disk.dm-0.out 5.18017578125 disk.drbd1965.out0 network.sent290971 disk.drbd1552.in0 disk.drbd1289.out0 disk.drbd2118.in0 network.recv63651 disk.rbd0.out0 disk.dm-0.in0) disk./.usagePercent32.34325808560657 disk.drbd1673.out0 disk.drbd1828.in0 disk.drbd1828.out0 disk.drbd1550.out0 disk.drbd1510.in0 disk.drbd2118.out0 disk.nvme0n1.in0 disk.drbd2041.in0 disk.drbd1584.out0* proc.memory.percent17.182111740112305 proc.cpu.threads29 disk.drbd1356.in0 disk.drbd1965.in0 disk.drbd1673.in0 disk.drbd1938.out0 disk.drbd2111.out0 disk.drbd1931.in0 disk.drbd2111.in0 disk.drbd1817.out0! proc.memory.rssMB 11260.46875 disk.sda.in0 disk.drbd1356.out0 disk.sda2.in0* proc.memory.availableMB38525.60546875 cpu15.776859923061583 disk.drbd1938.in0 disk.drbd1226.out0 disk.drbd2078.in0 disk.rbd1.out0 disk.sda2.out 4.89453125 disk.drbd1817.in0 disk.drbd1552.out0 disk.drbd1584.in0 disk.drbd2041.out0 disk.rbd0.in0 disk.sda.out 4.89453125 disk.sda1.in0 disk.drbd1510.out0 disk.drbd2078.out0 disk.drbd1550.in0 disk.drbd1226.in0" disk.nvme0n1.out 5.18017578125 disk.rbd1.in0 ^0+ _wandbruntime162  _runtime162"u _runtime 162.818049817 _step24 & lr-AdamW/pg18.994493083371828e-06 & lr-AdamW/pg28.994493083371828e-06 trainer/global_step349 % _timestamp1.7857954499950802e+09A  % _timestamp1.7857954499950802e+09 _runtime 162.818049817 _step24 & lr-AdamW/pg18.994493083371828e-06 & lr-AdamW/pg28.994493083371828e-06 trainer/global_step349z0+ _wandbruntime162  _runtime162&q'O O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated 6170523952932 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 306901924100 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated3702319 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed3692663 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 307656729600 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated3702319 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated 6320566462464 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed 5863604989472 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed6022290 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed 6320461993472 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 307656729600 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated2384930 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated 5863622028832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed 6320461993472 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed 6344296672768 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 trainer/global_step349 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed 6012892693504 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed 6012892693504 O WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated2329629 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed3692663 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed1004333 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 315981457920 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 307569299968 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current252 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated 6344460639232 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated2329629 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 _runtime 162.819301696 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated 6028340118528 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed 6170421110448 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed 6028315214848 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed3692663 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 307656729600 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated3702319 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated 6320566462464 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed 5863604989472 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed6022290 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed 6320461993472 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 307656729600 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated2384930 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated 5863622028832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed 6320461993472 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed 6344296672768 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 trainer/global_step349 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed 6012892693504 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed 6012892693504 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated 6170523952932 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 306901924100 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated3702319 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 H8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 308448919552 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated1007424 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed1007183 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed 6045522331648 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak528 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed 6188038755990 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed 5880345161792 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 _step26 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed 6338508376064 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated2391708 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed1384282 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current241 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated 5880362201152 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated6049128 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 N6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak538 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 trainer/global_step349 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated3712856 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 308448919552 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated2336272 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated 6338612852224 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current243 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated 6030076495872 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated3712856 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 307779397434 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed2336270 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated 6338612852224 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated 6045547235328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated2336272 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed2391465 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712魿&M ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed1007183 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed 6045522331648 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak528 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed 6188038755990 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed 5880345161792 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 _step26 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed 6338508376064 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated2391708 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed1384282 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current241 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated 5880362201152 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated6049128 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 N6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak538 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 trainer/global_step349 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated3712856 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 308448919552 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated2336272 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated 6338612852224 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current243 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated 6030076495872 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated3712856 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 307779397434 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed2336270 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated 6338612852224 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated 6045547235328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated2336272 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed2391465 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 308536356352 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated1384284 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated 6362567634432 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 317020399104 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 308448919552 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated1007424 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated2510+ _wandbruntime163  _runtime163 /loss/FRt1/assignment_loss5.594881612062454 0loss/FBt1/assignment_loss2.2095305238451277 .loss/FRt1/detection_loss4.113515603169799 /loss/FRt2/detection_loss3.6541073955194925 0loss/SRqqt2/detection_loss3.315858712401053 trainer/global_step349 _step27 1loss/SRqqt1/assignment_loss4.458026200532913 1loss/SRqqt2/assignment_loss4.518111065180615 0loss/FBt2/assignment_loss2.2125892639160156 .loss/FBt2/detection_loss24.10982201794783 epoch0 _runtime 163.189807739 1loss/SRqqt1/detection_loss3.1169407066817465 %loss/total_loss73.61203339571375 /loss/FRt2/assignment_loss5.916521036496726 .loss/FBt1/detection_loss10.39212925795998 $ _timestamp1.785795450366986e+09aӦ2 /loss/FRt1/assignment_loss5.594881612062454 0loss/FBt1/assignment_loss2.2095305238451277 .loss/FRt1/detection_loss4.113515603169799 /loss/FRt2/detection_loss3.6541073955194925 0loss/SRqqt2/detection_loss3.315858712401053 trainer/global_step349 _step27 1loss/SRqqt1/assignment_loss4.458026200532913 1loss/SRqqt2/assignment_loss4.518111065180615 0loss/FBt2/assignment_loss2.2125892639160156 .loss/FBt2/detection_loss24.10982201794783 epoch0 _runtime 163.189807739 1loss/SRqqt1/detection_loss3.1169407066817465 %loss/total_loss73.61203339571375 /loss/FRt2/assignment_loss5.916521036496726 .loss/FBt1/detection_loss10.39212925795998 $ _timestamp1.785795450366986e+09,U(: ᓲU gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp37.0 gpu.0.powerWatts115.973' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent38.65766666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0h1(:  U disk.rbd1.in0 disk.drbd2041.in0 disk.drbd2041.out0 disk.drbd1289.out0% memory_percent41.215306520462036 disk.sda.in0 disk.drbd1584.in0 disk.drbd1965.out0 disk.drbd1938.in0 disk.sda2.in0 disk.sda2.out 5.28515625 disk.drbd1552.out0 disk.drbd1938.out0 disk.drbd1285.in0 disk.drbd1550.in0 proc.cpu.threads29 network.recv66304 disk.dm-0.in0 disk.drbd1226.in0 disk.drbd1931.out0 disk.sda1.out0 disk.drbd1285.out0 disk.sda.out 5.28515625 disk.drbd2078.in0" disk.nvme0n1.out 5.34033203125 disk.drbd1673.out0 disk.sda1.in0! proc.memory.rssMB 11260.46875 disk.drbd1584.out0 disk.drbd2118.in0 disk.drbd1931.in0 cpu15.633085648408438 disk.drbd1510.out0 disk.drbd1356.in0 disk.drbd1817.out0 network.sent335934* disk./.usagePercent32.343259365633614 disk.dm-0.out 5.34033203125 disk.drbd2118.out0 disk.drbd1356.out0 disk.drbd1550.out0 disk.drbd1510.in0 disk.drbd2111.out0 disk.drbd1965.in0 disk.drbd1673.in0 disk.rbd1.out0 disk.drbd1552.in0 disk.drbd1828.out0 disk.drbd1289.in0 disk.nvme0n1.in0 disk.drbd1817.in0 disk.rbd0.out0$ disk./.usageGB578.3303260803223 disk.drbd2078.out0 disk.rbd0.in0* proc.memory.availableMB38525.13671875 disk.drbd2111.in0* proc.memory.percent17.182111740112305 disk.drbd1226.out0 disk.drbd1828.in0GU(: ʣU gpu.0.gpu41.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp37.0 gpu.0.powerWatts116.278' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent38.75933333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0dX,(:  U disk.dm-0.out 5.59228515625 disk.drbd1965.in0 disk.drbd1828.in0 disk.sda2.out 5.5859375 disk.drbd1817.in0 disk.drbd1285.in0 disk.sda.in0$ disk./.usageGB578.3301239013672 disk.nvme0n1.in0 disk.rbd1.out0 disk.drbd1285.out0$ memory_percent41.21502637863159 disk.drbd1965.out0 disk.rbd0.in0 disk.sda1.out0! proc.memory.rssMB 11260.46875 network.recv68796 disk.drbd1510.out0 disk.dm-0.in0 disk.drbd2041.out0 disk.drbd1938.out0 disk.drbd2118.out0 disk.drbd2111.in0 disk.rbd1.in0 disk.drbd1550.out0 disk.sda1.in0 disk.drbd2118.in0 disk.drbd1356.out0 disk.drbd2041.in0) disk./.usagePercent32.34324805872805 disk.drbd1931.in0 disk.drbd1828.out0 disk.drbd1289.out0 network.sent380947 disk.drbd1817.out0 disk.drbd1552.in0 disk.drbd1226.out0 disk.drbd1584.out0" disk.nvme0n1.out 5.59228515625 disk.sda2.in0 disk.drbd1289.in0 proc.cpu.threads29 disk.drbd1510.in0 disk.drbd1584.in0 disk.drbd1356.in0 disk.drbd1931.out0 disk.drbd1938.in0 disk.sda.out 5.5859375 disk.drbd2078.out0 disk.drbd1552.out0 disk.drbd1550.in0) proc.memory.availableMB 38525.3203125 disk.rbd0.out0 disk.drbd1226.in0 disk.drbd2078.in0 disk.drbd2111.out0 disk.drbd1673.in0 disk.drbd1673.out0* proc.memory.percent17.182111740112305 cpu15.505287708306351N0+ _wandbruntime185  _runtime185' _step28 ' lr-AdamW/pg11.0283102407637132e-05 ' lr-AdamW/pg21.0283102407637132e-05 trainer/global_step399 % _timestamp1.7857954724601164e+09 _runtime 185.283076823՘O ' lr-AdamW/pg11.0283102407637132e-05 ' lr-AdamW/pg21.0283102407637132e-05 trainer/global_step399 % _timestamp1.7857954724601164e+09 _runtime 185.283076823 _step28ƈ%0+ _wandbruntime185  _runtime185x O PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated2725860 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated2661779 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 HDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed 7222796279296 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated 7222900748288 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated 6888695958528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated 6871247883264 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed 7222796279296 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 _runtime 185.284258003 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed 6871230843904 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated4230321 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current257 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed6882442 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 351565435392 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed2661777 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current255 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 _step29 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated6892100 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated1577197 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed4220665 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed 7051318189956 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated 7222900748288 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 351652865024 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 LDDeviceh[+StatsMonitor.on_train_batch_start/segment.small_pool.allocated121 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated 7051421032440 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed2661777 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed1148408 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed2725603 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed 7249832906240 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed 6871230843904 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed1577195 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 % _timestamp1.7857954724610596e+09 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 351565435392 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 350704584484 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 361161851392 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated2661779 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 351652865024 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 350790387608 HDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed1577195 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 % _timestamp1.7857954724610596e+09 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 351565435392 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 350704584484 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 361161851392 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated2661779 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 351652865024 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 350790387608 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 HDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed 7222796279296 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated 7222900748288 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated 6888695958528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated 6871247883264 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed 7222796279296 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 _runtime 185.284258003 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed 6871230843904 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated4230321 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current257 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed6882442 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 351565435392 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed2661777 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current255 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 _step29 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated6892100 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated1577197 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed4220665 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed 7051318189956 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated 7222900748288 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 351652865024 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated 7051421032440 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed2661777 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed1148408 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed2725603 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed 7249832906240 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed 6871230843904 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed 7068935835498 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated2732643 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated 7268098485760 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated6909280 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 362062646272 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 362195410432 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _runtime 185.661868436 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed2668420 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated 6888414646272 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed2732382 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated 6905903075328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed6899608 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 _step30 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current261 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak528 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed 7240842661888 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed4231188 NDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 352445054976 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak538 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated1581134 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121*GM E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed6899608 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated 7240947138048 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 352532491776 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 351582057706 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated1151509 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated4240858 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 _step30 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current261 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak528 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed 7240842661888 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed4231188 NDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 352445054976 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak538 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated1581134 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed 7068935835498 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated2732643 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated 7268098485760 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated6909280 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 362062646272 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 XJDeviceSQd tatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 362195410432 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _runtime 185.661868436 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed2668420 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated 6888414646272 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed2732382 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated 6905903075328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed6899608 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 FDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated2993929 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 406493060608 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated 7537639260832 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed 7932215269464 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated 7932318111948 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed 7537622221472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated7752252 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed 7729568994304 HDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed2993927 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated4758323 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 _runtime 207.651555041 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed 8125130565120 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated 7749051798528 % _timestamp1.7857954948283296e+09 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed4748667 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed4748667 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 395649000448 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated2993929 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed 81251305651202x'O OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed4748667 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 395649000448 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated2993929 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed 8125130565120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 406362386432 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed7742594 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current255 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 _step33 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated 7729586033664 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated1291609 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated3065656 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed 7749026894848 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated2993929 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 406493060608 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated 7537639260832 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed 7932215269464 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated 7932318111948 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed 7537622221472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated7752252 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed 7729568994304 HDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed2993927 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated4758323 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 _runtime 207.651555041 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed 8125130565120 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated 7749051798528 % _timestamp1.7857954948283296e+09 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed4748667!0+ _wandbruntime208  _runtime208/M I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed4759190 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated7769432 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 395556324450 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated1294451 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated3000572 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated 8143281423872 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated4768860 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed3072184 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated 8173657834496 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated 7746752796672 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated3072435 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 396441190400 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed7759760 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated 8143281423872 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed 8143176947712 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated7769432 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 395470521214 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed7759760 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed1294202 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 % _timestamp1.7857954952012532e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated 7766258915328 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed 7746735757312 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed 7746735757312 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed 7949832915006 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed 7554362393792 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak538 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated1777984 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak528 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 _step34 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated 7949935757602 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 trainer/global_step449 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated4768860 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 396528627200 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 _runtime 208.024620144 A&M E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated 7949935757602 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 trainer/global_step449 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated4768860 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 396528627200 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 _runtime 208.024620144 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed4759190 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated7769432 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 395556324450 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated1294451 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated3000572 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated 8143281423872 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated4768860 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed3072184 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated 8173657834496 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated 7746752796672 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated3072435 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 396441190400 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed7759760 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated 8143281423872 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed 8143176947712 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated7769432 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 395470521214 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed7759760 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed1294202 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 % _timestamp1.7857954952012532e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated 7766258915328 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed 7746735757312 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed 7746735757312 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed 7949832915006 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed 7554362393792 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak538 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated1777984 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak528 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 _step34"0+ _wandbruntime208  _runtime208ر+ /loss/FRt2/assignment_loss5.934271436749083 1loss/SRqqt1/assignment_loss4.485264773879733 1loss/SRqqt2/assignment_loss4.596473788106164 1loss/SRqqt2/detection_loss3.7683048506462296 /loss/FBt2/detection_loss25.628875632371223 .loss/FRt1/assignment_loss5.57900255817478 0loss/FBt1/assignment_loss1.9857439157125112 .loss/FBt1/detection_loss9.632371786176353 trainer/global_step449 % _timestamp1.7857954952029085e+09 _runtime 208.025748078 _step35 /loss/FRt1/detection_loss5.6079948372760065 .loss/FRt2/detection_loss3.402194994266587 %loss/total_loss75.36650128784095 /loss/FBt2/assignment_loss1.989742398262024 1loss/SRqqt1/detection_loss2.7562603162202453 epoch0aE trainer/global_step449 % _timestamp1.7857954952029085e+09 _runtime 208.025748078 _step35 /loss/FRt1/detection_loss5.6079948372760065 .loss/FRt2/detection_loss3.402194994266587 %loss/total_loss75.36650128784095 /loss/FBt2/assignment_loss1.989742398262024 1loss/SRqqt1/detection_loss2.7562603162202453 epoch0 /loss/FRt2/assignment_loss5.934271436749083 1loss/SRqqt1/assignment_loss4.485264773879733 1loss/SRqqt2/assignment_loss4.596473788106164 1loss/SRqqt2/detection_loss3.7683048506462296 /loss/FBt2/detection_loss25.628875632371223 .loss/FRt1/assignment_loss5.57900255817478 0loss/FBt1/assignment_loss1.9857439157125112 .loss/FBt1/detection_loss9.632371786176353#{7ސ(: ӿU gpu.0.gpu24.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp37.0 gpu.0.powerWatts113.958' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent37.986 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0ܷ)(:  U disk.drbd1356.in0 network.sent429442) disk./.usagePercent32.34327536597168 disk.drbd1828.in0 disk.sda.in0 disk.drbd1285.in0 disk.drbd1510.in0 disk.drbd1817.in0 disk.dm-0.out 6.064453125 disk.drbd2111.out0 disk.sda1.out0 disk.drbd1289.out0 disk.drbd2078.out0% memory_percent41.215407848358154 disk.sda1.in0 disk.drbd1673.out0 disk.drbd1552.in0 disk.nvme0n1.in0 disk.sda2.in0 disk.rbd1.out0) proc.memory.availableMB 38525.0703125! proc.memory.rssMB 11260.46875 disk.drbd1931.in0 disk.drbd1226.out0 disk.rbd0.out0 disk.drbd1356.out0 disk.rbd1.in0$ disk./.usageGB578.3306121826172 disk.drbd1673.in0 disk.drbd1289.in0 disk.dm-0.in0 disk.drbd2118.in0 disk.sda2.out 6.3203125 disk.drbd2111.in0 disk.drbd1285.out0 disk.drbd2041.in0 disk.drbd1552.out0 disk.drbd1226.in0 disk.drbd1584.in0 disk.drbd1584.out0 disk.nvme0n1.out 6.064453125 disk.drbd1965.out0 disk.drbd1938.in0 disk.rbd0.in0 disk.sda.out 6.3203125 disk.drbd1965.in0 disk.drbd1931.out0 disk.drbd1938.out0 cpu15.333353916825219 proc.cpu.threads29 network.recv72894 disk.drbd1828.out0 disk.drbd1550.in0 disk.drbd2041.out0 disk.drbd1510.out0 disk.drbd2078.in0 disk.drbd2118.out0* proc.memory.percent17.182111740112305 disk.drbd1550.out0 disk.drbd1817.out0ŷ(: ꚗU gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts127.418' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.47266666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0-(:  U network.sent474443 disk.drbd1931.in0 disk.nvme0n1.in0 disk.sda2.in0 disk.drbd2111.out0& proc.memory.availableMB 38524.8125 disk.drbd1289.out0 disk.rbd0.in0 cpu15.26602471403447 disk.drbd1938.in0 disk.drbd1285.out0 disk.dm-0.in0 disk.rbd1.in0 proc.cpu.threads29 disk.drbd1552.in0 disk.drbd1510.in0 disk.drbd2078.out0 disk.drbd2111.in0 disk.rbd0.out0 disk.drbd1285.in0 disk.drbd1226.out0 disk.drbd1931.out0 disk.sda.out 6.6796875 disk.drbd1584.out0 disk.drbd1550.out0 disk.sda1.out0 disk.drbd1226.in0 disk.drbd1965.out0 disk.drbd1828.out0 disk.drbd1289.in0 disk.sda1.in0 disk.drbd1356.out0 disk.drbd1817.out0$ disk./.usageGB578.3306312561035 disk.drbd1965.in0 disk.drbd1552.out0 disk.drbd1550.in0 disk.sda.in0 disk.drbd1584.in0 disk.drbd2118.in0 disk.drbd1673.out0! proc.memory.rssMB 11260.46875 network.recv75675 disk.drbd2041.out0 disk.drbd2078.in0 disk.nvme0n1.out 6.263671875 disk.rbd1.out 0.5546875* proc.memory.percent17.182111740112305 disk.drbd2041.in0 disk.drbd1510.out0 disk.drbd1356.in0 disk.drbd1673.in0$ memory_percent41.21580123901367 disk.drbd1938.out0 disk.drbd2118.out0 disk.drbd1828.in0 disk.dm-0.out 6.263671875 disk.sda2.out 6.6796875* disk./.usagePercent32.343276432660886 disk.drbd1817.in00+ _wandbruntime229  _runtime229> ' lr-AdamW/pg11.2860321056167744e-05 ' lr-AdamW/pg21.2860321056167744e-05 trainer/global_step499 % _timestamp1.7857955171554277e+09 _runtime 229.978419654 _step367k  _step36 ' lr-AdamW/pg11.2860321056167744e-05 ' lr-AdamW/pg21.2860321056167744e-05 trainer/global_step499 % _timestamp1.7857955171554277e+09 _runtime 229.978419654$8S 0+ _wandbruntime229  _runtime229˯'O WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated3326079 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 438481511500 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 439557706240 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed3404751 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 451655597568 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 HDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed 9027464850944 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed 8587907144704 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 451516534784 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated5286325 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated1434138 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 trainer/global_step499 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed 8587907144704 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed1970895 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed 8609382734848 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed 9027464850944 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed1433856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 _step37 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 439645135872 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 438567314624 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated 8813215191456 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated 9027569319936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed 9060899269632 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed 8374630837472 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 HDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed 9027464850944 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed 8587907144704 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 451516534784 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated5286325 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated1434138 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 trainer/global_step499 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed 8587907144704 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed1970895 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed 8609382734848 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed 9027464850944 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed1433856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 _step37 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 439645135872%0+ _wandbruntime230  _runtime230 !&M H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed 9045511233536 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 440437325824 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 439444787958 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated3332722 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed3332720 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 $ _timestamp1.785795517526815e+09 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated 8605090947072 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 trainer/global_step499 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated3332722 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 452418435584 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated 9045615709696 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed1436720 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed 8391371009792 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated 9045615709696 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated5296862 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed3332720 N6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 439358984722 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak538 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed 8830729994514 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed3411552 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated3411796 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated5296862 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 440524762624 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 440437325824 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed 8605073907712 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated 8391388049152 _step38 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated 8830832837110 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated 9079168052224 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated8629584 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 440524762624 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated1974834 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak528 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated8629584 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed1974832 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed5287192 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current244 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated 8626614755328 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed 9079008287232 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed8619912 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 439358984722 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak538 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed 8830729994514 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed3411552 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated3411796 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated5296862 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 440524762624 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 440437325824 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed 8605073907712 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated 8391388049152 _step38 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated 8830832837110 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated 9079168052224 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated8629584 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 440524762624 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated1974834 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak528 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated8629584 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed1974832 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed5287192 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current244 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated 8626614755328 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed 9079008287232 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed8619912 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated5296862 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed3332720 Ne2 0loss/SRqqt1/assignment_loss4.42808259101141 1loss/SRqqt2/assignment_loss4.491520140197251 /loss/FBt2/assignment_loss1.793149721622467 .loss/FRt1/detection_loss5.132135976401586 %loss/total_loss82.61566837328262 _step39 /loss/FRt2/detection_loss3.4404010295566887 .loss/FBt2/detection_loss36.06080426871777 /loss/FRt2/assignment_loss5.847083168800431 0loss/FBt1/assignment_loss1.7258449925316706 1loss/SRqqt2/detection_loss3.5619129628598034 trainer/global_step499 % _timestamp1.7857955175285285e+09 _runtime 230.351368723 /loss/FRt1/assignment_loss5.467056622580876 0loss/SRqqt1/detection_loss2.973068199058374 .loss/FBt1/detection_loss7.694608699944284 epoch0' 4(:  ǧU disk.dm-0.in0 disk.drbd1226.in0 disk.sda2.out7.015625 disk.drbd1817.in0 disk.drbd1289.out0% memory_percent41.215676069259644 network.recv76853 disk.drbd1931.in0 disk.drbd1552.out0 disk.drbd1550.in0 disk.rbd0.in0 disk.drbd1510.in0 disk.drbd1673.out0 disk.sda2.in0 disk.drbd2111.out0 disk.rbd1.out 0.6328125 disk.drbd1510.out0 disk.drbd1817.out0 disk.drbd1965.in0 disk.drbd2041.in0! proc.memory.rssMB 11260.46875$ disk./.usageGB578.3302459716797 disk.drbd1550.out0 disk.drbd1226.out0* proc.memory.percent17.182111740112305 disk.drbd1931.out0 disk.drbd1584.in0 disk.drbd2118.in0 disk.rbd1.in0 disk.drbd1965.out0 disk.drbd1673.in0 disk.drbd1828.in0 disk.drbd1828.out0 disk.sda1.out0 disk.drbd1285.in0 disk.drbd1356.out0 disk.drbd2118.out0 disk.drbd2041.out0 disk.drbd1584.out0 disk.sda1.in0 cpu15.208925700988198" disk.nvme0n1.out 6.54736328125 disk.drbd1356.in0 network.sent477683 disk.sda.out7.015625 disk.dm-0.out 6.54736328125* proc.memory.availableMB38524.89453125 disk.drbd2078.in0 disk.drbd2078.out0 disk.drbd1552.in0) disk./.usagePercent32.34325488553896 disk.nvme0n1.in0 disk.drbd2111.in0 disk.rbd0.out0 proc.cpu.threads29 disk.drbd1289.in0 disk.drbd1938.in0 disk.drbd1938.out0 disk.sda.in0 disk.drbd1285.out0(: ǧݯV gpu.0.gpu38.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp37.0 gpu.0.powerWatts95.611' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.870333333333335 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0;H|0+ _wandbruntime252  _runtime252١K ' lr-AdamW/pg11.4148930380433047e-05 ' lr-AdamW/pg21.4148930380433047e-05 trainer/global_step549 % _timestamp1.7857955396784277e+09 _runtime 252.501383788 _step406 trainer/global_step549 % _timestamp1.7857955396784277e+09 _runtime 252.501383788 _step40 ' lr-AdamW/pg11.4148930380433047e-05 ' lr-AdamW/pg21.4148930380433047e-05(k0+ _wandbruntime252  _runtime252;O NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated 9446262334464 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated3745308 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated1577561 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 482369975008 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated 9446262334464 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed 9446245295104 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 483641271296 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed5804671 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated 9469763478528 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated3658229 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed3658227 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed9462898 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed3658227 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated 9929903605760 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed 9929799136768 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _step41 HDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 496873106432 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed 9966481007104 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated5814327 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated 9694112270964 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated9472556 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated5814327 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed 9211639453472 $ _timestamp1.785795539679382e+09 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 483641271296 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0{V-G'O O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated 9446262334464 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated3745308 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated1577561 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 482369975008 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated 9446262334464 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed 9446245295104 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 483641271296 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed5804671 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated 9469763478528 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated3658229 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed3658227 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed9462898 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed3658227 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 TBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated 9929903605760 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed 9929799136768 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _step41 HDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 496873106432 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed 9966481007104 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated5814327 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated 9694112270964 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated9472556 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated5814327 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed 9211639453472 $ _timestamp1.785795539679382e+09 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 483641271296)/0+ _wandbruntime252  _runtime252J&M I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed3664870 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak538 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 484520898048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 _runtime 252.872827501 NDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated3664872 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated 9947949995520 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated9489736 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated2171684 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 497777654272 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated5824864 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed 9228379625792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 % _timestamp1.7857955400494516e+09 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed5815194 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated 9228396665152 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated 9947949995520 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed 9463412058112 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated 9486970595328 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed3664870 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated 9463429097472 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed1580133 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed 9947845519360 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed 9711627074022 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 _step42 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed9480064 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed 9984592678912 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 484433461248 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated3664872 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated 9711729916618 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed5815194 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed 9463412058112 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated1580385 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated9489736 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current252 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed3751815 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 trainer/global_step549 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed9480064 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current254 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 484520898048 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 484433461248 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0M F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated 9711729916618 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed5815194 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed 9463412058112 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated1580385 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated9489736 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current252 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed3751815 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 trainer/global_step549 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed9480064 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current254 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 484520898048 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 484433461248 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed3664870 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak538 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 484520898048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 _runtime 252.872827501 NDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated3664872 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated 9947949995520 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated9489736 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated2171684 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 497777654272 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated5824864 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed 9228379625792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 % _timestamp1.7857955400494516e+09 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed5815194 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated 9228396665152 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated 9947949995520 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed 9463412058112 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated 9486970595328 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed3664870 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated 9463429097472 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed1580133 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed 9947845519360 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed 9711627074022 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 _step42 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed9480064 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed 9984592678912 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 484433461248 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated3664872 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0*q:0+ _wandbruntime252  _runtime25250 /loss/FRt2/assignment_loss5.940405933602342 .loss/FRt1/detection_loss5.388346390646012 1loss/SRqqt2/detection_loss3.4011646888178326 /loss/FBt1/detection_loss12.384146647656273 _step43 /loss/FRt1/assignment_loss5.583669388880495 1loss/SRqqt1/assignment_loss4.385130920864287 0loss/FBt1/assignment_loss1.3782141825248455 /loss/FBt2/assignment_loss1.523740331331889 -loss/FRt2/detection_loss3.28405751664083 &loss/total_loss110.51993050100664 trainer/global_step549 0loss/SRqqt2/assignment_loss4.49460053184758 epoch0 _runtime 252.874055989 1loss/SRqqt1/detection_loss2.9350102366081305 /loss/FBt2/detection_loss59.821443731586136 % _timestamp1.7857955400512226e+09׌  0loss/SRqqt2/assignment_loss4.49460053184758 epoch0 _runtime 252.874055989 1loss/SRqqt1/detection_loss2.9350102366081305 /loss/FBt2/detection_loss59.821443731586136 % _timestamp1.7857955400512226e+09 /loss/FRt2/assignment_loss5.940405933602342 .loss/FRt1/detection_loss5.388346390646012 1loss/SRqqt2/detection_loss3.4011646888178326 /loss/FBt1/detection_loss12.384146647656273 _step43 /loss/FRt1/assignment_loss5.583669388880495 1loss/SRqqt1/assignment_loss4.385130920864287 0loss/FBt1/assignment_loss1.3782141825248455 /loss/FBt2/assignment_loss1.523740331331889 -loss/FRt2/detection_loss3.28405751664083 &loss/total_loss110.51993050100664 trainer/global_step549+̄(: ֧ѣU gpu.0.gpu41.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp37.0 gpu.0.powerWatts96.177' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent32.059000000000005 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0C7(:  ֧U network.recv79946 disk.drbd1356.in0 disk.drbd1828.in0 disk.rbd0.out0 disk.drbd1965.in0 disk.sda.out 7.47265625 network.sent522961 disk.sda1.out0 disk.drbd1510.out0 disk.drbd2078.out0 disk.drbd1938.out0 disk.rbd0.in0 proc.cpu.threads29) disk./.usagePercent32.34328133943122 disk.drbd2078.in0 disk.drbd1817.in0 disk.drbd1552.out0$ memory_percent41.21703505516052 disk.sda1.in0 disk.dm-0.in0 disk.drbd2118.out0$ disk./.usageGB578.3307189941406 disk.drbd1285.in0 disk.drbd1510.in0 disk.drbd1289.out0 disk.drbd1550.in0 disk.drbd1289.in0 disk.drbd1938.in0 disk.drbd1285.out0 disk.drbd1226.out0 disk.drbd1584.in0 disk.drbd1584.out0 disk.drbd1356.out0 disk.drbd1673.out0 disk.drbd2041.out0 disk.drbd1226.in0 disk.drbd2118.in0 disk.drbd1817.out0 disk.rbd1.in0 disk.drbd1931.out0 disk.nvme0n1.in0" disk.nvme0n1.out 6.70361328125 disk.drbd2111.out0 disk.drbd1965.out0 disk.drbd1931.in0 disk.drbd1552.in0* proc.memory.availableMB38524.00390625 disk.drbd2111.in0 disk.rbd1.out 0.6328125 disk.drbd1673.in0 disk.drbd1550.out0* proc.memory.percent17.182111740112305 cpu15.157182829929608 disk.dm-0.out 6.70361328125 disk.sda2.in0 disk.sda2.out 7.47265625 disk.drbd2041.in0 disk.drbd1828.out0 disk.sda.in0! proc.memory.rssMB 11260.46875lC/ǜ(: U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts125.969' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent41.989666666666665 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0\ul6(:  إV disk.sda1.in0 disk.drbd1510.in0 disk.drbd1285.in0 disk.drbd2078.in0 disk.drbd1356.out0 disk.drbd1965.out0 disk.drbd1552.in0 disk.drbd1226.in0 disk.drbd1226.out0 disk.drbd2118.out0! disk.nvme0n1.out 7.1513671875 disk.sda2.out 7.8203125 network.recv82588) disk./.usagePercent32.34328240612043 disk.drbd1550.out0 disk.rbd0.out0 disk.drbd1584.in0 disk.drbd1356.in0* proc.memory.availableMB38523.78515625 disk.rbd0.in0 disk.drbd1584.out0 cpu15.111149360617764 network.sent567877 disk.drbd1931.out0 disk.drbd1828.out0 disk.sda.out 7.8203125 disk.drbd1285.out0 disk.drbd1673.in0 disk.drbd2041.out0 disk.drbd1550.in0 disk.drbd1938.out0 disk.nvme0n1.in0 disk.drbd2111.out0 disk.drbd2118.in0 disk.drbd1817.out0 disk.drbd2041.in0 disk.drbd1289.out0 disk.drbd1965.in0 disk.drbd2111.in0# disk./.usageGB578.330738067627 disk.drbd1817.in0 disk.drbd1673.out0 disk.dm-0.in0 disk.drbd1552.out0 disk.dm-0.out 7.1513671875 disk.sda2.in0 disk.rbd1.out 0.7509765625 disk.drbd2078.out0* proc.memory.percent17.182111740112305% memory_percent41.217368841171265 disk.drbd1289.in0 disk.rbd1.in0 disk.drbd1931.in0 disk.sda.in0! proc.memory.rssMB 11260.46875 proc.cpu.threads29 disk.drbd1828.in0 disk.drbd1938.in0 disk.sda1.out0 disk.drbd1510.out00+ _wandbruntime274  _runtime274%SR ' lr-AdamW/pg21.5437539704698353e-05 trainer/global_step599 % _timestamp1.7857955617904222e+09 _runtime 274.613378636 _step44 ' lr-AdamW/pg11.5437539704698353e-05`=A ' lr-AdamW/pg11.5437539704698353e-05 ' lr-AdamW/pg21.5437539704698353e-05 trainer/global_step599 % _timestamp1.7857955617904222e+09 _runtime 274.613378636 _step44,T0+ _wandbruntime274  _runtime274'O IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed10832133422592 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 526258438516 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated6342329 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 542055262208 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated10304600484864 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed6332673 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak538 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed2364595 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 527637406720 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated10332708 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current249 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 527549977088 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated10048665108832 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated1721297 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed10574906507988 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed3990377 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated10872174580736 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated10832237891584 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated10304600484864 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed6332673 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak538 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed2364595 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 527637406720 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated10332708 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current249 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 527549977088 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated10048665108832 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated1721297 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed10574906507988 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed3990377 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated10872174580736 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated10832237891584 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed10832133422592 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 526258438516 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032-㐻0+ _wandbruntime274  _runtime274 tfM WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 M>DevI&iceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current252 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated10890288072192 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated10850284281344 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated10349888 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed10890128307200 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 527135911738 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed10340216 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated10850284281344 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed3997020 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current254 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated3997022 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated10349888 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 527221714974 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 528517033472 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed4092409 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 trainer/global_step599 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated6352866 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed10065388241792 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed6343196 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed1723877 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated10592626996126 _step46 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed10340216 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed6343196 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated10347326435328 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed10321750208512 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated10321767247872 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 % _timestamp1.7857955621634824e+09 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 542961636864 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed10592524153530 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated10065405281152 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 _runtime 274.986911125U9&M REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated10349888 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed10890128307200 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 527135911738 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed10340216 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated10850284281344 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed3997020 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current254 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated3997022 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated10349888 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 527221714974 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 528517033472 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed4092409 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 trainer/global_step599 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated6352866 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed10065388241792 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed6343196 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed1723877 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated10592626996126 _step46 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed10340216 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed6343196 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated10347326435328 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed10321750208512 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated10321767247872 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 % _timestamp1.7857955621634824e+09 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 542961636864 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed10592524153530 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated10065405281152 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 _runtime 274.986911125 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current252 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated10890288072192 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated10850284281344 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0.VQ0+ _wandbruntime274  _runtime274B,L* 1loss/SRqqt1/assignment_loss4.210893507987734 /loss/FRt2/detection_loss3.4619839387584705 /loss/FBt2/detection_loss29.582490732272465 epoch0 1loss/SRqqt2/assignment_loss4.210696464174249 1loss/SRqqt1/detection_loss2.2286910463183647 trainer/global_step599 _step47 /loss/FRt1/assignment_loss5.541065387427807 0loss/FBt1/assignment_loss1.0451880872249604 /loss/FRt1/detection_loss5.0212071212008595 /loss/SRqqt2/detection_loss3.61853236880865 % _timestamp1.7857955621652906e+09 _runtime 274.988129865 .loss/FRt2/assignment_loss5.91597884528491 0loss/FBt2/assignment_loss1.1039303193489711 /loss/FBt1/detection_loss11.761502704505498 %loss/total_loss77.70216052331294) 0loss/FBt1/assignment_loss1.0451880872249604 /loss/FRt1/detection_loss5.0212071212008595 /loss/SRqqt2/detection_loss3.61853236880865 % _timestamp1.7857955621652906e+09 _runtime 274.988129865 .loss/FRt2/assignment_loss5.91597884528491 0loss/FBt2/assignment_loss1.1039303193489711 /loss/FBt1/detection_loss11.761502704505498 %loss/total_loss77.70216052331294 1loss/SRqqt1/assignment_loss4.210893507987734 /loss/FRt2/detection_loss3.4619839387584705 /loss/FBt2/detection_loss29.582490732272465 epoch0 1loss/SRqqt2/assignment_loss4.210696464174249 1loss/SRqqt1/detection_loss2.2286910463183647 trainer/global_step599 _step47 /loss/FRt1/assignment_loss5.541065387427807/N͐(: U gpu.0.gpu38.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts118.773' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent39.591 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0N4(:  սU disk.drbd1356.out0 disk.sda2.out 8.27734375 disk.drbd1931.out0 disk.drbd1552.in0 network.recv82984 disk.drbd2041.out0 disk.drbd1552.out0 disk.drbd2118.out0 disk.nvme0n1.out 7.43359375 proc.cpu.threads29 disk.drbd1817.out0 disk.drbd1931.in0 disk.drbd1938.out0 disk.rbd0.in0 disk.drbd2078.in0 disk.drbd2118.in0 disk.drbd1226.in0 network.sent570888$ disk./.usageGB578.3307571411133 disk.rbd1.out 1.2900390625 disk.drbd1938.in0 disk.sda1.in0) disk./.usagePercent32.34328347280963 disk.drbd1817.in0 disk.drbd1673.out0 disk.dm-0.out 7.43359375 disk.drbd1584.in0* proc.memory.availableMB38523.19140625 disk.drbd1356.in0 disk.sda2.in0 disk.drbd1673.in0 disk.drbd1226.out0* proc.memory.percent17.182111740112305 disk.sda.in0 disk.drbd1285.out0 disk.drbd2078.out0 disk.nvme0n1.in0 disk.drbd1828.out0 disk.drbd1550.in0 disk.drbd1289.in0 disk.drbd1510.out0 disk.drbd1584.out0 disk.drbd1965.out0 disk.drbd1289.out0 disk.sda1.out0 disk.drbd1285.in0 cpu15.064820469454617 disk.drbd2111.in0! proc.memory.rssMB 11260.46875 disk.drbd2111.out0 disk.drbd1965.in0 disk.drbd1550.out0 disk.rbd0.out0 disk.drbd1510.in0 disk.rbd1.in0 disk.drbd2041.in0 disk.drbd1828.in0 disk.sda.out 8.27734375$ memory_percent41.21827483177185 disk.dm-0.in0e{[0+ _wandbruntime296  _runtime296B % _timestamp1.7857955841324875e+09 _runtime 296.955451145 _step48 ' lr-AdamW/pg11.6726149028963658e-05 ' lr-AdamW/pg21.6726149028963658e-05 trainer/global_step649\;՛ trainer/global_step649 % _timestamp1.7857955841324875e+09 _runtime 296.955451145 _step48 ' lr-AdamW/pg11.6726149028963658e-05 ' lr-AdamW/pg21.6726149028963658e-050jk0+ _wandbruntime296  _runtime296I"O G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 587060417024 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated10885673724832 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated2561447 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 571633542144 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 _runtime 296.956798322 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated11734572177408 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed6860675 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed2561445 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed11183202 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak538 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed11777510671872 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 571633542144 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak528 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated11777668346880 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed11162921595904 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed11734467708416 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed4426566 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed1865121 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 570232705148 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed4322527 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed11162921595904 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated4322529 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 571546112512 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed11190450254848 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated4426813 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current247 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated11734572177408 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed11734467708416 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated11162938635264 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 571546112512 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 _step49 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated11162938635264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated11192860 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated11190475158528 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed11455803587496 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 trainer/global_step649 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current245 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 587193188352 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed11734467708416 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated11162938635264 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 571546112512 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 _step49 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated11162938635264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated11192860 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated11190475158528 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed11455803587496 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 trainer/global_step649 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current245 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 587193188352 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 587060417024 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated10885673724832 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated2561447 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 571633542144 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 _runtime 296.956798322 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated11734572177408 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed6860675 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed2561445 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed11183202 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak538 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed11777510671872 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 571633542144 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak528 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated11777668346880 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed11162921595904 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed11734467708416 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed4426566 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed1865121 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 570232705148 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed4322527 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed11162921595904 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed01c0+ _wandbruntime297  _runtime297Mϯ&M TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 % _timestamp1.7857955845061684e+09 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed11752514091008 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated11210040 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated4433538 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated6880868 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 572513168896 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak528 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated4329172 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 572425732096 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed11473421233038 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 572425732096 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed4329170 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated10902413897152 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed11795626128384 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated11210040 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated2565384 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 587968756736 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 _runtime 297.329583746 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 572513168896 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed4329170 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed6871198 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed10902396857792 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated1868154 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated11795779601920 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated11752618567168 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed11200368 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed4433296 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated11207682275328 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 trainer/global_step649 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated11180105398272 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak538 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 588097326592 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed11180088358912 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated11752618567168 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 571024375246 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 _step50 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251LM&M _runtime 297.329583746 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 572513168896 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed4329170 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed6871198 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed10902396857792 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated1868154 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated11795779601920 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated11752618567168 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed11200368 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed4433296 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated11207682275328 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 trainer/global_step649 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated11180105398272 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak538 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 588097326592 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed11180088358912 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated11752618567168 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 571024375246 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 _step50 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 % _timestamp1.7857955845061684e+09 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed11752514091008 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated11210040 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated4433538 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated6880868 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 572513168896 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak528 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated4329172 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 572425732096 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed11473421233038 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 572425732096 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed4329170 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated10902413897152 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed11795626128384 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated11210040 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated2565384 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 587968756736 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed02Ě0+ _wandbruntime297  _runtime297Ȣ /loss/FRt2/detection_loss3.1761642889875286 /loss/SRqqt1/detection_loss2.55374604440947 /loss/FBt2/detection_loss25.125118611114367 % _timestamp1.7857955845082927e+09 2loss/SRqqt1/assignment_loss3.6468914000787467 2loss/SRqqt2/assignment_loss3.6425218778113795 0loss/SRqqt2/detection_loss4.562798776446956 .loss/FBt1/detection_loss9.144487039871617 _runtime 297.331193681 /loss/FRt1/assignment_loss5.589366810662406 /loss/FRt2/assignment_loss5.948127535154235 /loss/FBt2/assignment_loss0.827975121991975 epoch0 trainer/global_step649 _step51 0loss/FBt1/assignment_loss0.8635951810412936 .loss/FRt1/detection_loss5.738075163215399 %loss/total_loss70.81886785078538 2loss/SRqqt1/assignment_loss3.6468914000787467 2loss/SRqqt2/assignment_loss3.6425218778113795 0loss/SRqqt2/detection_loss4.562798776446956 .loss/FBt1/detection_loss9.144487039871617 _runtime 297.331193681 /loss/FRt1/assignment_loss5.589366810662406 /loss/FRt2/assignment_loss5.948127535154235 /loss/FBt2/assignment_loss0.827975121991975 epoch0 trainer/global_step649 _step51 0loss/FBt1/assignment_loss0.8635951810412936 .loss/FRt1/detection_loss5.738075163215399 %loss/total_loss70.81886785078538 /loss/FRt2/detection_loss3.1761642889875286 /loss/SRqqt1/detection_lossf0 l2.55374604440947 /loss/FBt2/detection_loss25.125118611114367 % _timestamp1.7857955845082927e+093uH(: ᗢU gpu.0.gpu24.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts117.523' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent39.17433333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0{+(:  U disk.drbd2118.out0 disk.drbd1817.in0 network.sent615981 disk.drbd1938.in0 disk.drbd2078.in0 disk.nvme0n1.out8.96875 disk.drbd1931.in0 disk.drbd2041.in0 disk.drbd1552.out0 disk.drbd1550.in0 disk.sda.in0 disk.drbd2111.out0 disk.drbd1828.out0 disk.sda2.in0 disk.drbd2111.in0 disk.drbd1828.in0 disk.drbd1285.out0 disk.drbd1226.out0# disk./.usageGB578.330738067627 disk.dm-0.out8.96875 disk.drbd2041.out0 disk.drbd2078.out0 cpu15.025313482233932 disk.drbd1552.in0 disk.drbd1510.out0 disk.drbd1584.out0 disk.drbd1817.out0 disk.rbd1.out 1.4619140625 disk.drbd1510.in0( proc.memory.availableMB 38523.265625 disk.drbd1289.in0 disk.rbd0.in0 disk.drbd1584.in0 disk.rbd0.out0 disk.sda.out 8.68359375 disk.sda1.out0 disk.drbd2118.in0 disk.nvme0n1.in0 disk.drbd1356.out0 disk.sda2.out 8.68359375 proc.cpu.threads29 disk.drbd1965.in0) disk./.usagePercent32.34328240612043 disk.drbd1673.out0 disk.drbd1550.out0 disk.drbd1285.in0* proc.memory.percent17.182111740112305 network.recv85564 disk.drbd1938.out0! proc.memory.rssMB 11260.46875 disk.rbd1.in0 disk.drbd1673.in0 disk.drbd1931.out0 disk.drbd1289.out0 disk.drbd1356.in0$ memory_percent41.21816158294678 disk.drbd1965.out0 disk.dm-0.in0 disk.drbd1226.in0 disk.sda1.in0u(: U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp37.0 gpu.0.powerWatts121.462' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent40.48733333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0z,(:  V disk.rbd0.out0 disk.sda1.out0 disk.drbd2078.in0 disk.drbd1550.out0 disk.drbd1817.out0 disk.drbd1828.in0) disk./.usagePercent32.34327920605281 disk.sda1.in0% memory_percent41.218167543411255 disk.drbd1584.in0 disk.drbd1584.out0 disk.drbd2041.out0 disk.drbd1938.out0 disk.sda.in0 disk.drbd2111.out0 disk.drbd1938.in0 network.recv87942# disk./.usageGB578.330680847168 disk.drbd1285.out0 disk.rbd1.in0 cpu14.982026845634964 disk.drbd1510.out0 disk.dm-0.in0 disk.sda2.in0 disk.drbd1817.in0 disk.drbd1285.in0 disk.drbd2078.out0 disk.drbd2111.in0 disk.drbd2041.in0* proc.memory.availableMB38523.26171875 disk.drbd1510.in0 disk.drbd1931.out0 disk.nvme0n1.in0 proc.cpu.threads29 disk.rbd0.in0 disk.drbd1226.in0 disk.drbd1226.out0 disk.drbd1965.in0 disk.drbd1673.in0 disk.drbd1673.out0 disk.drbd1828.out0 disk.drbd2118.out0 disk.drbd1356.in0 disk.drbd1552.in0 disk.drbd1289.out0 disk.nvme0n1.out9.1875 disk.drbd1356.out0 disk.sda2.out 9.00390625 disk.rbd1.out 1.4619140625 disk.drbd1552.out0 disk.drbd1550.in0! proc.memory.rssMB 11260.46875* proc.memory.percent17.182111740112305 disk.drbd1931.in0 disk.drbd1289.in0 network.sent661031 disk.sda.out 9.00390625 disk.dm-0.out9.1875 disk.drbd2118.in0 disk.drbd1965.out0<"db0+ _wandbruntime319  _runtime319 ' lr-AdamW/pg21.8014758353228962e-05 trainer/global_step699 $ _timestamp1.785795606449269e+09 _runtime 319.272242479 _step52 ' lr-AdamW/pg11.8014758353228962e-05Q ' lr-AdamW/pg11.8014758353228962e-05 ' lr-AdamW/pg21.8014758353228962e-05 trainer/global_step699 $ _timestamp1.785795606449269e+09 _runtime 319.272242479 _step5240+ _wandbruntime319  _runtime319A֥'O bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 632376158208 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed12683043190272 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 615629677568 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed12043354 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated7398333 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated11722682340832 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated2007271 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak528 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated12636906463232 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated12050830998528 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed12050806094848 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 614121168656 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 % _timestamp1.7857956064502523e+09 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed2758295 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated12683207156736 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 trainer/global_step699 _step53 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated4654679 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated7398333 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 615542247936 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed12636801994240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated12021276785664 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136'O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed12683043190272 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 615629677568 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed12043354 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated7398333 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated11722682340832 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated2007271 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak528 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated12636906463232 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated12050830998528 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed12050806094848 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 614121168656 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 % _timestamp1.7857956064502523e+09 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed2758295 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated12683207156736 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 trainer/global_step699 _step53 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated4654679 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated7398333 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 615542247936 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed12636801994240 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated12021276785664 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 632376158208 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed05c.0+ _wandbruntime319  _runtime319}2@M G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed12060520 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 633145238528 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 614912838754 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed12038426509312 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated2762234 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 633271711232 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed2762232 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed11739405473792 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated12070192 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated2010025 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated12038443548672 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed7399200 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 151376384 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 614998641990 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated4772259 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated7408870 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 616509304320 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 _runtime 319.645233104 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated12654952852992 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated12068038115328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 126472704 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current245 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 trainer/global_step699 _step54 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated12038443548672 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed4772012 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed12068013211648 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated11739422513152 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed4661320 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed12060520 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed12354318312546 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 616509304320 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 UHDeviceStatsMonitor.on_train_batchYN|n_end/allocated_bytes.small_pool.current87436800 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed7399200 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current247 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated12354421155142 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated4661322 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated7408870 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed4661320 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed12038426509312 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed12701158450176 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed7399200 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current247 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated12354421155142 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated4661322 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated7408870 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed4661320 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed12038426509312 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed12701158450176 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated12070192 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated2010025 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated12038443548672 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed7399200 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 151376384 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 614998641990 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated4772259 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated7408870 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 616509304320 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 _runtime 319.645233104 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated12654952852992 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated12068038115328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 126472704 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current245 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 trainer/global_step699 _step54 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated12038443548672 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed4772012 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed12068013211648 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated11739422513152 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed4661320 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed12060520 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed12354318312546 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 616509304320 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc2516y0+ _wandbruntime319  _runtime319J2j 0loss/FBt1/assignment_loss0.7830592785562788 0loss/SRqqt1/detection_loss2.235816861252749 1loss/SRqqt2/detection_loss3.8197639613997105 trainer/global_step699 % _timestamp1.7857956068235166e+09 0loss/SRqqt2/assignment_loss3.05349830139515 /loss/FRt2/detection_loss3.2673157424029737 epoch0 /loss/FRt1/assignment_loss5.536877338702862 /loss/FRt2/assignment_loss5.941698909986137 .loss/FBt1/detection_loss9.720155161652448 .loss/FBt2/detection_loss50.39445017491068 %loss/total_loss93.59759515903806 _runtime 319.646337666 _step55 1loss/SRqqt1/assignment_loss3.039688083860609 0loss/FBt2/assignment_loss0.8528695106506348 .loss/FRt1/detection_loss4.952401834267836=U0 .loss/FBt1/detection_loss9.720155161652448 .loss/FBt2/detection_loss50.39445017491068 %loss/total_loss93.59759515903806 _runtime 319.646337666 _step55 1loss/SRqqt1/assignment_loss3.039688083860609 0loss/FBt2/assignment_loss0.8528695106506348 .loss/FRt1/detection_loss4.952401834267836 0loss/FBt1/assignment_loss0.7830592785562788 0loss/SRqqt1/detection_loss2.235816861252749 1loss/SRqqt2/detection_loss3.8197639613997105 trainer/global_step699 % _timestamp1.7857956068235166e+09 0loss/SRqqt2/assignment_loss3.05349830139515 /loss/FRt2/detection_loss3.2673157424029737 epoch0 /loss/FRt1/assignment_loss5.536877338702862 /loss/FRt2/assignment_loss5.9416989099861377,UW(: U gpu.0.gpu21.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp37.0 gpu.0.powerWatts94.103' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.367666666666665 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0Ab.(:  ˬV disk.drbd1285.out0 cpu14.927529822391401# disk./.usageGB578.330738067627 disk.drbd1673.in0 disk.drbd1289.out0 disk.sda2.in0 disk.drbd1817.out0 disk.drbd1552.out0 disk.drbd1938.in0 disk.rbd0.out0 network.sent664244 disk.drbd1356.in0 disk.drbd1552.in0 disk.drbd2078.in0 disk.drbd1965.in0 disk.drbd1550.in0 disk.drbd1510.in0! proc.memory.rssMB 11260.46875 disk.drbd2111.in0 disk.drbd2041.in0 disk.sda.out 9.3359375& proc.memory.availableMB 38523.4375 network.recv89279 disk.drbd1356.out0 disk.drbd2118.in0 disk.rbd1.in0 disk.sda.in0 disk.drbd1931.out0 disk.drbd2078.out0 disk.drbd1289.in0 disk.drbd1285.in0 disk.drbd1510.out0 disk.dm-0.in0 proc.cpu.threads29) disk./.usagePercent32.34328240612043 disk.sda1.in0% memory_percent41.217899322509766 disk.nvme0n1.in0 disk.drbd1828.in0 disk.drbd1938.out0 disk.sda1.out0* proc.memory.percent17.182111740112305 disk.dm-0.out 9.43359375 disk.drbd1226.out0 disk.nvme0n1.out 9.43359375 disk.sda2.out 9.3359375 disk.drbd1673.out0 disk.drbd1828.out0 disk.drbd1584.in0 disk.drbd2111.out0 disk.drbd1817.in0 disk.drbd2041.out0 disk.drbd1550.out0 disk.drbd2118.out0 disk.rbd1.out 1.5244140625 disk.rbd0.in0 disk.drbd1931.in0 disk.drbd1226.in0 disk.drbd1584.out0 disk.drbd1965.out0$~0+ _wandbruntime341  _runtime3414E ' lr-AdamW/pg11.9303367677494267e-05 ' lr-AdamW/pg21.9303367677494267e-05 trainer/global_step749 % _timestamp1.7857956289327805e+09 _runtime 341.75575946 _step56v0 _step56 ' lr-AdamW/pg11.9303367677494267e-05 ' lr-AdamW/pg21.9303367677494267e-05 trainer/global_step749 % _timestamp1.7857956289327805e+09 _runtime 341.755759468+I0+ _wandbruntime341  _runtime341s'O WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated2150965 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated12559690956832 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed2150698 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 677582898688 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 _step57 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated7926335 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed13539136280064 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed13217597746512 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 trainer/global_step749 _runtime 341.757055066 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 659625812992 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed5105843 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed12903506 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed4986827 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed7916679 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated13539240749056 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current269 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated12911186838528 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed12879597896704 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed7916679 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 659538383360 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed13588609965056 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated12913164 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed12559673917472 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 657923829040 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated12879614936064 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed4986827 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated13588769737216 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed13539136280064 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated7926335 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated13217700588996)O RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed12879597896704 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed7916679 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 659538383360 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed13588609965056 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated12913164 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed12559673917472 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 657923829040 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated12879614936064 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed4986827 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated13588769737216 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed13539136280064 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated7926335 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated13217700588996 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated2150965 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated12559690956832 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed2150698 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 677582898688 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 _step57 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated7926335 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed13539136280064 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed13217597746512 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 trainer/global_step749 _runtime 341.757055066 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 659625812992 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed5105843 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed12903506 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed4986827 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed7916679 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated13539240749056 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current269 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated12911186838528 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed09jH0+ _wandbruntime342  _runtime342D:&M UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated7936872 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current241 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed13557182662656 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated2959084 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated4993472 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed12928369051648 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed12896764659712 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated4993472 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 660505439744 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed7927202 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current243 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated12928393955328 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed13235215392054 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 658801302262 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 trainer/global_step749 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 _step58 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed2959082 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated12930344 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed4993470 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated12930344 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed13606727832064 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated5112896 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 658887105498 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed12896764659712 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed5112653 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated7936872 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated13557287138816 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 149279232 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated12896781699072 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360{ѩ&M ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated12896781699072 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated7936872 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current241 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed13557182662656 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated2959084 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated4993472 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed12928369051648 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed12896764659712 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated4993472 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 660505439744 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed7927202 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current243 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated12928393955328 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed13235215392054 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 658801302262 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 trainer/global_step749 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 _step58 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed2959082 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated12930344 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed4993470 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated12930344 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed13606727832064 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated5112896 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 658887105498 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed12896764659712 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed5112653 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated7936872 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated13557287138816 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 149279232 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0:'J0+ _wandbruntime342  _runtime342k 0loss/SRqqt1/assignment_loss2.75065814507635 0loss/FBt1/assignment_loss0.7194246286931245 /loss/FBt1/detection_loss14.940298387214032 _step59 2loss/SRqqt2/assignment_loss2.8678467930032965 -loss/FRt1/detection_loss4.98482828354463 /loss/FRt2/detection_loss2.9591661614498923 .loss/FBt2/detection_loss43.98153740726411 epoch0 trainer/global_step749 % _timestamp1.7857956293089569e+09 _runtime 342.131777386 /loss/FRt1/assignment_loss5.549929305911064 0loss/FBt2/assignment_loss0.6916192024946213 0loss/SRqqt2/detection_loss3.662364080632001 /loss/FRt2/assignment_loss5.972620772463935 1loss/SRqqt1/detection_loss1.7146784932519312 %loss/total_loss90.79497166099898U %loss/total_loss90.79497166099898 0loss/SRqqt1/assignment_loss2.75065814507635 0loss/FBt1/assignment_loss0.7194246286931245 /loss/FBt1/detection_loss14.940298387214032 _step59 2loss/SRqqt2/assignment_loss2.8678467930032965 -loss/FRt1/detection_loss4.98482828354463 /loss/FRt2/detection_loss2.9591661614498923 .loss/FBt2/detection_loss43.98153740726411 epoch0 trainer/global_step749 % _timestamp1.7857956293089569e+09 _runtime 342.131777386 /loss/FRt1/assignment_loss5.549929305911064 0loss/FBt2/assignment_loss0.6916192024946213 0loss/SRqqt2/detection_loss3.662364080632001 /loss/FRt2/assignment_loss5.972620772463935 1loss/SRqqt1/detection_loss1.7146784932519312;I]M(: 쬙U gpu.0.gpu1.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp37.0 gpu.0.powerWatts94.31' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.436666666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0M2(:  U disk.sda1.in0 disk.dm-0.in0* proc.memory.percent17.182111740112305 cpu14.87701536434463) disk./.usagePercent32.34328347280963 disk.sda2.out 9.91796875 disk.drbd2111.out0 disk.drbd1817.in0 proc.cpu.threads29 disk.drbd1552.out0 disk.drbd1550.in0 disk.drbd1938.out0 disk.sda1.out0 disk.dm-0.out 9.61328125 disk.nvme0n1.out 9.61328125 disk.drbd2041.in0 disk.drbd1289.in0 disk.drbd1938.in0 disk.drbd1931.out0 disk.rbd0.in0 disk.drbd1510.in0 disk.sda2.in0 disk.drbd2111.in0$ memory_percent41.21905565261841! proc.memory.rssMB 11260.46875 disk.drbd1931.in0 disk.drbd1828.out0 disk.drbd1584.in0 disk.drbd1965.in0 disk.drbd1828.in0 disk.drbd1226.in0 disk.drbd1673.in0 disk.drbd2118.in0 disk.drbd2118.out0 disk.drbd1356.in0 disk.drbd1356.out0 disk.drbd1817.out0 network.sent709223 network.recv91799 disk.sda.out 9.91796875 disk.drbd1285.in0 disk.drbd2078.out0 disk.nvme0n1.in0) proc.memory.availableMB 38522.6796875 disk.rbd1.out 1.5244140625 disk.drbd2041.out0 disk.drbd1552.in0 disk.drbd1965.out0 disk.drbd1289.out0 disk.drbd2078.in0 disk.drbd1584.out0$ disk./.usageGB578.3307571411133 disk.rbd1.in0 disk.rbd0.out0 disk.drbd1226.out0 disk.drbd1673.out0 disk.drbd1550.out0 disk.drbd1510.out0 disk.drbd1285.out0 disk.sda.in0Z a(: U gpu.0.gpu38.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts120.235' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent40.07833333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0R2(:  ťV disk.drbd1673.out0 disk.rbd0.out0 disk.drbd1817.out0 disk.rbd1.in0 disk.drbd1931.out0 disk.drbd1938.out0 disk.drbd2111.in0 disk.drbd1584.out0 disk.sda2.out 10.2109375 disk.drbd1965.in0 disk.drbd2041.in0 disk.drbd1828.in0 disk.dm-0.in0 disk.dm-0.out 9.85546875 disk.drbd1356.in0 network.recv95033 disk.rbd0.in0 disk.nvme0n1.in0 disk.sda2.in0 cpu14.837929180207762# disk./.usageGB578.330738067627 disk.drbd2041.out0 disk.drbd1510.out0 disk.drbd1584.in0 disk.drbd1965.out0 network.sent754449 disk.drbd1550.out0 disk.sda.in0 disk.sda.out 10.2109375 disk.drbd1510.in0 disk.nvme0n1.out 9.85546875 disk.drbd2111.out0 disk.drbd1817.in0 proc.cpu.threads29 disk.sda1.out0) disk./.usagePercent32.34328240612043 disk.drbd1828.out0 disk.drbd1550.in0* proc.memory.percent17.182111740112305 disk.drbd1931.in0 disk.drbd1289.out0* proc.memory.availableMB38523.55859375 disk.drbd1673.in0 disk.sda1.in0 disk.drbd2078.in0 disk.drbd2118.out0 disk.drbd1356.out0 disk.rbd1.out 1.650390625! proc.memory.rssMB 11260.46875 disk.drbd1289.in0 disk.drbd1552.out0 disk.drbd2078.out0 disk.drbd1285.in0$ memory_percent41.21771454811096 disk.drbd1285.out0 disk.drbd1226.in0 disk.drbd1226.out0 disk.drbd1552.in0 disk.drbd1938.in0 disk.drbd2118.in00+ _wandbruntime363  _runtime363׻ _runtime 363.74333597 _step60 & lr-AdamW/pg12.059197700175957e-05 & lr-AdamW/pg22.059197700175957e-05 trainer/global_step799 % _timestamp1.7857956509203546e+09K trainer/global_step799 % _timestamp1.7857956509203546e+09 _runtime 363.74333597 _step60 & lr-AdamW/pg12.059197700175957e-05 & lr-AdamW/pg22.059197700175957e-05<+H+h0+ _wandbruntime363  _runtime363 O dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated13771542678528 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 703621948416 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 703534518784 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated5446512 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak538 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated14441575034880 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed13737936047104 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated13396699572832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed13737936047104 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 703534518784 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed14441470565888 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated14494351911424 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 _runtime 363.744698614 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed13763658 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed5446243 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated13773316 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated2294515 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed13771517774848 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 _step61 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed8444681 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated8454337 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated14098597668504 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed5318977 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak528 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed14494192139264 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed8444681 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed14441470565888 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed2294248 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated13737953086464 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed5318977 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated8454337 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed3151995 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 722674364416 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed13396682533472 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 % _timestamp1.7857956509213588e+09 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 701812292548 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 722809232896 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 701898095672 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121Ͷ 'O % _timestamp1.7857956509213588e+09 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 701812292548 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 722809232896 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 701898095672 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated13771542678528 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 703621948416 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 703534518784 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated5446512 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak538 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated14441575034880 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed13737936047104 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated13396699572832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed13737936047104 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 703534518784 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed14441470565888 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated14494351911424 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 _runtime 363.744698614 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed13763658 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed5446243 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated13773316 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated2294515 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed13771517774848 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 _step61 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed8444681 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated8454337 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated14098597668504 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed5318977 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak528 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed14494192139264 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed8444681 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed14441470565888 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed2294248 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated13737953086464 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed5318977 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated8454337 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed3151995 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 722674364416 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed13396682533472 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121=lK0+ _wandbruntime364  _runtime364a&M ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated13755119849472 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 _step62 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated14459621424640 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed14116112471562 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated14116215314158 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 723577158144 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 702775569006 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 704501575168 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated5325622 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed13413422705792 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed13780824 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed13780824 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated14512461814784 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak538 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed13755102810112 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 704414138368 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated14459621424640 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated13788749795328 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 trainer/global_step799 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated3155934 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated5325622 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed3155932 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed5325620 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed5453071 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated8464874 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current239 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 723712019456 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated13790496 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated8464874 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated5453312 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak528 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed13755102810112 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed8455204 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed14512302049792 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed8455204 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 704501575168 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current241 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated13790496 % _timestamp1.7857956512926052e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 704414138368 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed5325620 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed13788724891648 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251Ks^M VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated14459621424640 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed14116112471562 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated14116215314158 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 723577158144 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 702775569006 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 704501575168 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated5325622 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed13413422705792 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed13780824 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed13780824 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated14512461814784 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak538 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed13755102810112 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 704414138368 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated14459621424640 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated13788749795328 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 trainer/global_step799 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated3155934 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated5325622 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 L=>lUDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed3155932 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed5325620 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed5453071 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated8464874 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current239 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 723712019456 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated13790496 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated8464874 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated5453312 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak528 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed13755102810112 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed8455204 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed14512302049792 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed8455204 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 704501575168 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current241 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated13790496 % _timestamp1.7857956512926052e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 704414138368 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed5325620 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed13788724891648 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated13755119849472 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 _step62>k0+ _wandbruntime364  _runtime364Ƃb .loss/FRt1/assignment_loss5.51985701918602 1loss/SRqqt1/assignment_loss2.563208285821687 0loss/FBt2/assignment_loss0.6496909175600324 epoch0 _step63 0loss/FRt2/assignment_loss5.9055610123802635 1loss/SRqqt2/assignment_loss2.780673210322857 /loss/FRt1/detection_loss4.9649137100204825 /loss/FBt1/detection_loss13.630763109109097 .loss/FBt2/detection_loss50.12108813864844 %loss/total_loss95.70283006032872 % _timestamp1.7857956512945018e+09 _runtime 364.117362274 1loss/FBt1/assignment_loss0.48836710474764305 /loss/FRt2/detection_loss3.2334585832614526 1loss/SRqqt1/detection_loss1.6285508207966404 0loss/SRqqt2/detection_loss4.216698148474097 trainer/global_step799g 0loss/FBt2/assignment_loss0.6496909175600324 epoch0 _step63 0loss/FRt2/assignment_loss5.9055610123802635 1loss/SRqqt2/assignment_loss2.780673210322857 /loss/FRt1/detection_loss4.9649137100204825 /loss/FBt1/detection_loss13.630763109109097 .loss/FBt2/detection_loss50.12108813864844 %loss/total_loss95.70283006032872 % _timestamp1.7857956512945018e+09 _runtime 364.117362274 1loss/FBt1/assignment_loss0.48836710474764305 /loss/FRt2/detection_loss3.2334585832614526 1loss/SRqqt1/detection_loss1.6285508207966404 0loss/SRqqt2/detection_loss4.216698148474097 trainer/global_step799 .loss/FRt1/assignment_loss5.51985701918602 1loss/SRqqt1/assignment_loss2.563208285821687?Nϛ(: ΨU gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts121.564' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent40.52133333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0Px0(:  ΨU disk.drbd1356.in0 disk.drbd1817.out0 network.recv95471# disk./.usageGB578.330680847168 disk.drbd1931.in0 disk.drbd1550.in0 disk.drbd1285.out0 disk.drbd1584.in0) disk./.usagePercent32.34327920605281 disk.drbd1289.in0 disk.drbd1938.out0 disk.drbd1285.in0 disk.drbd1226.in0 disk.nvme0n1.out 10.078125 disk.drbd1965.in0 disk.drbd1550.out0 disk.drbd1931.out0 disk.drbd2118.out0 disk.drbd1356.out0 cpu14.813741670214966 disk.drbd2111.out0 disk.drbd1817.in0 disk.sda1.in0 disk.sda2.out 10.7109375 disk.rbd1.in0 disk.drbd1510.in0 disk.drbd1226.out0 disk.rbd1.out 1.650390625 disk.drbd1965.out0$ memory_percent41.21786952018738 disk.drbd1552.out0 disk.rbd0.out0 disk.drbd2111.in0 disk.drbd2118.in0 disk.drbd1510.out0 disk.drbd1828.in0 disk.drbd1289.out0 disk.sda1.out0 disk.drbd2078.out0 disk.drbd1584.out0 disk.dm-0.in0! proc.memory.rssMB 11260.46875 disk.dm-0.out 10.078125* proc.memory.availableMB38523.45703125 disk.drbd1673.in0 disk.drbd2041.out0 disk.drbd1552.in0 disk.rbd0.in0 disk.sda.in0 disk.sda.out 10.7109375* proc.memory.percent17.182111740112305 disk.drbd1938.in0 disk.nvme0n1.in0 disk.drbd1673.out0 network.sent757499 disk.drbd2041.in0 disk.drbd1828.out0 disk.drbd2078.in0 disk.sda2.in0 proc.cpu.threads290+ _wandbruntime386  _runtime386lj trainer/global_step849 % _timestamp1.7857956735573187e+09 _runtime 386.380272434 _step64 ' lr-AdamW/pg12.1880586326024876e-05 ' lr-AdamW/pg22.1880586326024876e-05ce_ ' lr-AdamW/pg22.1880586326024876e-05 trainer/global_step849 % _timestamp1.7857956735573187e+09 _runtime 386.380272434 _step64 ' lr-AdamW/pg12.1880586326024876e-05@B=0+ _wandbruntime386  _runtime386'O GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated8982339 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated15343909320704 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 747618083840 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 747530654208 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed14596274197504 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 trainer/global_step849 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed8972683 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed15343804851712 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed5651127 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated8982339 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated14233708188832 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed14623810 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated14633468 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 768018363904 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated14631898518528 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 _step65 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed14596274197504 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed8972683 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak528 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated14979494748012 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 % _timestamp1.7857956735579987e+09 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed5651127 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated5787875 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed14233691149472 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current254 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current256 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 745700756056 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated14596291236864 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak538 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed3348845 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed2438774 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed14631873614848 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated14596291236864 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed5787619 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated2439028 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed14979391905528 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated5651129 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed15343804851712 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed15343804851712 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed5651127 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated8982339 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated14233708188832 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed14623810 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated14633468 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 768018363904 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated14631898518528 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 _step65 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed14596274197504 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed8972683 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak528 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated14979494748012 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 % _timestamp1.7857956735579987e+09 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed5651127 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated5787875 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed14233691149472 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current254 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current256 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 745700756056 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated14596291236864 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak538 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed3348845 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed2438774 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed14631873614848 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated14596291236864 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed5787619 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated2439028 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed14979391905528 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated5651129 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed15343804851712A0+ _wandbruntime386  _runtime386?WM G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated14650648 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed8983206 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed14613440960512 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak528 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 768916988928 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated8992876 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed5794383 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated14649105635328 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed15361851234304 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated15361955710464 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated14650648 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed14250431321792 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed14649080731648 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed5657770 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allkocated15418022624256 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed2441601 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current243 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated14997112393666 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed14997009551070 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 trainer/global_step849 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 748497710592 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current245 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 768784224768 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed14640976 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated3352784 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed15417864956416 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 $ _timestamp1.785795673936858e+09 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed14613440960512 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed8983206 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed3352782 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated14613457999872 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 748497710592 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated14250448361152 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak538 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _runtime 386.760260008 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936*s&M M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed2441601 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current243 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated14997112393666 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed14997009551070 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 trainer/global_step849 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 748497710592 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current245 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 768784224768 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed14640976 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated3352784 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed15417864956416 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 $ _timestamp1.785795673936858e+09 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed14613440960512 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed8983206 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed3352782 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated14613457999872 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 748497710592 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated14250448361152 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak538 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _runtime 386.760260008 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated14650648 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed8983206 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed14613440960512 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak528 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 768916988928 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated8992876 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed5794383 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated14649105635328 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed15361851234304 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated15361955710464 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated14650648 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed14250431321792 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed14649080731648 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed5657770 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated15418022624256 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160B70+ _wandbruntime386  _runtime386 /loss/FRt2/assignment_loss5.943828732879074 /loss/FBt2/detection_loss26.311894737757168 %loss/total_loss63.62348484756793 /loss/FBt2/assignment_loss0.675945153603187 .loss/FRt1/detection_loss5.019786269869655 0loss/SRqqt2/detection_loss3.549526776308599 trainer/global_step849 % _timestamp1.7857956739385777e+09 _step67 /loss/FRt1/assignment_loss5.551812089979649 2loss/SRqqt2/assignment_loss2.7601551361705945 1loss/SRqqt1/detection_loss2.3117280000286553 epoch0 1loss/SRqqt1/assignment_loss2.451599117029797 1loss/FBt1/assignment_loss0.26337285144015765 /loss/FRt2/detection_loss3.0537007402766636 .loss/FBt1/detection_loss5.730135242224731 _runtime 386.76140035-Ş /loss/FRt2/assignment_loss5.943828732879074 /loss/FBt2/detection_loss26.311894737757168 %loss/total_loss63.62348484756793 /loss/FBt2/assignment_loss0.675945153603187 .loss/FRt1/detection_loss5.019786269869655 0loss/SRqqt2/detection_loss3.549526776308599 trainer/global_step849 % _timestamp1.7857956739385777e+09 _step67 /loss/FRt1/assignment_loss5.551812089979649 2loss/SRqqt2/assignment_loss2.7601551361705945 1loss/SRqqt1/detection_loss2.3117280000286553 epoch0 1loss/SRqqt1/assignment_loss2.451599117029797 1loss/FBt1/assignment_loss0.26337285144015765 /loss/FRt2/detection_loss3.0537007402766636 .loss/FBt1/detection_loss5.730135242224731 _runtime 386.76140035C:`:(:  ݨU disk.drbd1550.out0 disk.drbd1226.out0 disk.drbd1584.out0 disk.drbd1673.out0 disk.drbd1226.in0 disk.drbd1584.in0 disk.drbd1552.out0 disk.rbd0.out0 network.recv98335 disk.drbd1285.in0 disk.drbd2118.in0 disk.sda2.out 11.08203125 disk.drbd1289.in0 proc.cpu.threads29 disk.sda1.in0 disk.dm-0.in0 disk.nvme0n1.in0 disk.rbd1.out 1.712890625! proc.memory.rssMB 11260.46875 disk.drbd1938.out0 disk.sda.out 11.08203125 disk.rbd1.in0$ disk./.usageGB578.3307418823242 disk.drbd2111.in0 disk.drbd1673.in0 disk.drbd1552.in0 disk.drbd1828.out0$ memory_percent41.21756553649902) disk./.usagePercent32.34328261945827 disk.drbd1931.in0 disk.drbd1931.out0 disk.drbd1828.in0 disk.sda.in0* proc.memory.percent17.182111740112305 disk.rbd0.in0 disk.dm-0.out10.32568359375 disk.drbd2078.in0 disk.drbd2078.out0 disk.drbd1356.in0 disk.drbd2041.in0 disk.drbd2041.out0 network.sent802790 disk.drbd1510.in0# disk.nvme0n1.out10.32568359375 disk.drbd2111.out0 disk.drbd1817.out0 disk.drbd1965.in0 disk.drbd1938.in0' proc.memory.availableMB 38523.65625 disk.drbd1510.out0 disk.drbd1289.out0 cpu14.788096149886035 disk.drbd1285.out0 disk.drbd1965.out0 disk.drbd1550.in0 disk.drbd2118.out0 disk.drbd1817.in0 disk.drbd1356.out0 disk.sda2.in0 disk.sda1.out0<0(: ݨV gpu.0.gpu34.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts95.459' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent31.81966666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0t(: рU gpu.0.gpu41.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts94.379' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent31.45966666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0gk?(:  U disk.drbd1226.in0 disk.drbd1226.out0 disk.sda2.in0! proc.memory.rssMB 11260.46875 disk.drbd2041.out0 disk.rbd0.in0 disk.drbd1510.out0 cpu14.766454240806455 disk.drbd1510.in0# disk.nvme0n1.out10.51708984375 disk.drbd1938.out0 disk.drbd1965.out0 disk.drbd1673.in0 disk.drbd1550.out0 disk.drbd1289.in0 disk.dm-0.out10.51708984375 disk.drbd1584.in0 disk.drbd2111.out0 disk.drbd1552.in0 disk.rbd0.out0 disk.drbd1584.out0 disk.drbd2118.in0 network.sent847922 disk.sda1.out0$ disk./.usageGB578.3307838439941 disk.drbd1938.in0 disk.drbd1550.in0 disk.drbd2118.out0 disk.drbd2111.in0 disk.drbd1828.in0 disk.sda.in0 disk.drbd2078.in0* proc.memory.percent17.182111740112305 disk.drbd1931.out0 disk.drbd1289.out0 disk.sda2.out 11.48046875 disk.drbd1817.out0 proc.cpu.threads29 disk.drbd2041.in0 disk.sda1.in0 disk.drbd1285.out0 disk.dm-0.in0$ memory_percent41.21871590614319 network.recv100927 disk.drbd1965.in0 disk.drbd1931.in0 disk.rbd1.in0 disk.drbd1673.out0 disk.drbd2078.out0 disk.nvme0n1.in0 disk.drbd1817.in0 disk.drbd1828.out0 disk.drbd1285.in0* disk./.usagePercent32.343284966174515 disk.drbd1552.out0 disk.drbd1356.in0 disk.drbd1356.out0 disk.rbd1.out 1.712890625 disk.sda.out 11.48046875* proc.memory.availableMB38522.90234375C[0+ _wandbruntime408  _runtime408 _runtime 408.548393185 _step68 ' lr-AdamW/pg12.3169195650290183e-05 ' lr-AdamW/pg22.3169195650290183e-05 trainer/global_step899 % _timestamp1.7857956957253697e+09F/ ' lr-AdamW/pg12.3169195650290183e-05 ' lr-AdamW/pg22.3169195650290183e-05 trainer/global_step899 % _timestamp1.7857956957253697e+09 _runtime 408.548393185 _step68Dc0+ _wandbruntime408  _runtime408qů'O VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 791526789632 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated16305511479808 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 _step69 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated15070716804832 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed9500685 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated16246243606528 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed16246139137536 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated2582723 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed15860288985036 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed6128161 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak528 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 791526789632 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 791614219264 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current259 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated9510341 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 " _timestamp1.7857956957264e+09 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 791614219264 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated15860391827520 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed15454612347904 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed5983277 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 813257121280 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed15483962 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed16246139137536 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed5983277 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed15070699765472 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated16246243606528 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated9510341 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated15454629387264 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated5983279 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed2582466 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 789675022688 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated3545697 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed15492229454848 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 813122252800 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current257 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated15493620 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed9500685 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated15492254358528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992rO GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed15070699765472 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated16246243606528 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated9510341 _LD-m8%eviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated15454629387264 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated5983279 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed2582466 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 789675022688 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated3545697 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed15492229454848 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 813122252800 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current257 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated15493620 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed9500685 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated15492254358528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 791526789632 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated16305511479808 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 _step69 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated15070716804832 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed9500685 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated16246243606528 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed16246139137536 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated2582723 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed15860288985036 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed6128161 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak528 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 791526789632 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 791614219264 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current259 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated9510341 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 " _timestamp1.7857956957264e+09 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 791614219264 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated15860391827520 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed15454612347904 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed5983277 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 813257121280 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed15483962 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed16246139137536 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed5983277EN0+ _wandbruntime408  _runtime408^&M RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed9511208 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed15877906630578 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed15087439937792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 792406409216 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed5989920 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 _step70 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current256 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed3549632 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak528 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated15878009473174 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak538 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated9520878 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 792406409216 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated6135204 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated15087456977152 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 790552496022 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated15471796150272 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated9520878 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed16323467239936 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed15509436571648 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 % _timestamp1.7857956961119561e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated15510800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 792493846016 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated5989922 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated16264289996288 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated15471796150272 trainer/global_step899 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed9511208 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed5989920 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed6134948 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 792493846016 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed15501128 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed15501128 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 814159238144 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed15471779110912 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 790466692786 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated15510800 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated3549634 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated15509461475328 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed16264185520128 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated2585570 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed15501128 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed15501128 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 814159238144 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed15471779110912 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 790466692786 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated15510800 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated3549634 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated15509461475328 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed16264185520128 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated2585570 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 792406409216 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed5989920 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 _step70 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current256 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed3549632 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak528 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated15878009473174 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak538 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated9520878 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 792406409216 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated6135204 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated15087456977152 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 790552496022 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated15471796150272 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated9520878 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed16323467239936 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed15509436571648 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 % _timestamp1.7857956961119561e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated15510800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 792493846016 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated5989922 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated16264289996288 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated15471796150272 trainer/global_step899 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed9511208 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed5989920 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed6134948 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 792493846016F0+ _wandbruntime408  _runtime408 trainer/global_step899 _runtime 408.936787962 .loss/FRt1/detection_loss4.657000825247344 .loss/FRt2/detection_loss4.173571395653266 /loss/SRqqt1/detection_loss1.23861375168952 .loss/FBt1/detection_loss8.667969658566523 % _timestamp1.7857956961139545e+09 /loss/FRt2/assignment_loss5.889486542454472 1loss/SRqqt1/assignment_loss2.393730429040284 0loss/FBt2/assignment_loss0.5812638662755489 0loss/SRqqt2/detection_loss3.582549583652745 %loss/total_loss68.41397033889164 _step71 /loss/FRt1/assignment_loss5.552352274165434 2loss/SRqqt2/assignment_loss2.7372074217899987 0loss/FBt1/assignment_loss0.2623657707923225 /loss/FBt2/detection_loss28.677858819564182 epoch0 epoch0 trainer/global_step899 _runtime 408.936787962 .loss/FRt1/detection_loss4.657000825247344 .loss/FRt2/detection_loss4.173571395653266 /loss/SRqqt1/detection_loss1.23861375168952 .loss/FBt1/detection_loss8.667969658566523 % _timestamp1.7857956961139545e+09 /loss/FRt2/assignment_loss5.889486542454472 1loss/SRqqt1/assignment_loss2.393730429040284 0loss/FBt2/assignment_loss0.5812638662755489 0loss/SRqqt2/detection_loss3.582549583652745 %loss/total_loss68.41397033889164 _step71 /loss/FRt1/assignment_loss5.552352274165434 2loss/SRqqt2/assignment_loss2.7372074217899987 0loss/FBt1/assignment_loss0.2623657707923225 /loss/FBt2/detection_loss28.677858819564182G(: ȒU gpu.0.gpu33.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts121.818' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent40.605999999999995 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0NRy(:  U disk.rbd1.out 1.9248046875 disk.drbd1965.in0 disk.drbd1289.out0 disk.drbd1938.in0 disk.drbd1226.in0 disk.drbd1510.in0 disk.drbd2078.in0 disk.drbd1356.out0! proc.memory.rssMB 11260.46875 disk.sda2.in0$ memory_percent41.21896028518677) proc.memory.availableMB 38522.7421875 disk.drbd1673.out0 disk.drbd2111.out0 disk.drbd1817.out0 disk.drbd1931.out0 disk.drbd1828.out0 disk.drbd2118.in0 disk.sda.out 11.89453125 disk.drbd1285.in0! disk.nvme0n1.out 10.798828125$ disk./.usageGB578.3307456970215 disk.sda1.out0 disk.drbd1584.out0 disk.drbd1356.in0 disk.drbd1965.out0* proc.memory.percent17.182111740112305) disk./.usagePercent32.34328283279611 disk.drbd1289.in0 disk.rbd0.in0 disk.sda1.in0 disk.drbd2118.out0 disk.drbd1817.in0 network.sent851153 disk.drbd2078.out0 disk.drbd1584.in0 disk.sda2.out 11.89453125 network.recv102256 disk.drbd1931.in0 disk.drbd1552.in0 disk.drbd1550.in0 disk.dm-0.out 10.798828125 disk.drbd2041.in0 disk.rbd0.out0 disk.drbd1226.out0 cpu14.7302417122933 disk.drbd2041.out0 disk.drbd1552.out0 disk.drbd1285.out0 disk.nvme0n1.in0 disk.drbd1673.in0 disk.drbd1550.out0 proc.cpu.threads29 disk.drbd1828.in0 disk.drbd1938.out0 disk.drbd1510.out0 disk.dm-0.in0 disk.rbd1.in0 disk.sda.in0 disk.drbd2111.in0Z%90+ _wandbruntime430  _runtime430یNl ' lr-AdamW/pg22.4457804974555488e-05 trainer/global_step949 $ _timestamp1.785795717664417e+09 _runtime 430.487381851 _step72 ' lr-AdamW/pg12.4457804974555488e-05r9 trainer/global_step949 $ _timestamp1.785795717664417e+09 _runtime 430.487381851 _step72 ' lr-AdamW/pg12.4457804974555488e-05 ' lr-AdamW/pg22.4457804974555488e-05Hթ0+ _wandbruntime430  _runtime430A]֩'O [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed15907708381472 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed16352585294848 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated10038343 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak538 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated10038343 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed10028687 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated17211109753344 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated15907725420832 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated16353772 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current247 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 trainer/global_step949 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated16312967537664 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated3742547 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 858499554816 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed3742545 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 _step73 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed17148473423360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed16312950498304 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated6315429 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated17148577892352 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 835522925056 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 % _timestamp1.7857957176653862e+09 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated16312967537664 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 858364686336 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated6315429 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated17148577892352 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed17148473423360 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed6315427 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed6469946 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current245 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed17210949981184 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated16352610198528 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed6315427 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed2727401 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 835522925056 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed6315427 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed6469946 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current245 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed17210949981184 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated16352610198528 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed6315427 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed2727401 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 835522925056 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed16352585294848 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated10038343 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak538 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated10038343 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed10028687 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated17211109753344 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated15907725420832 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated16353772 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current247 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 trainer/global_step949 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated16312967537664 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated3742547 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 858499554816 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed3742545 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 _step73 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed17148473423360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed16312950498304 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated6315429 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated17148577892352 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 835522925056 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 % _timestamp1.7857957176653862e+09 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated16312967537664 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 858364686336 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated6315429 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated17148577892352 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed17148473423360 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated16370952 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 859268962816 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed6476741 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed2730259 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated2730506 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated10048880 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed10039210 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated6476990 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed17229061374464 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed16369792411648 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 859401726976 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 836489981440 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated6322072 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak528 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 trainer/global_step949 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251h3M % _timestamp1.7857957180353634e+09 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak538 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated16369817315328 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed6322070 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated16330134300672 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated10048880 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 834355156294 _runtime 430.858738033 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed10039210 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 REDeviceStatsMonitor .on_train_batch_end/active_bytes.large_pool.current17039360 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated2730506 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated10048880 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed10039210 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated6476990 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed17229061374464 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed16369792411648 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 859401726976 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 836489981440 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated6322072 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak528 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 trainer/global_step949 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed17166519805952 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current249 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated3746484 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated17166624282112 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 836402544640 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed16361280 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated6322072 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed16330117261312 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated15924465593152 _step74 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated16370952 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 859268962816 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed6476741 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed2730259 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ODeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated16744734036832 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak538 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed6809594 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated17213924 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed6647577 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated17171305688064 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed17171288648704 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed16744716997472 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 % _timestamp1.7857957399877422e+09 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed10556689 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed18050807709184 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated2870460 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0nPZO UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed17212941134848 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 903550050304 trainer/global_step999 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated6647579 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed17204266 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 879606490112 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed10556689 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed18050807709184 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated2870460 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 877366146580 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed3939395 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated18116653054464 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated6809857 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current263 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 _step77 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 879519060480 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated3939397 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 877451949704 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed10556689 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak528 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated10566345 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated16744734036832 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak538 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed6809594 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated17213924 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed6647577 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated17171305688064 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed17171288648704 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed16744716997472 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 % _timestamp1.7857957399877422e+09 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 I6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed6654220 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed6816361 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed17221432 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed10567212 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 _runtime 453.197054731 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed17221432 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 880398680064 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated17188472451072 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 904456489472 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed17188455411712 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed10567212 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak538 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed3943332 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated6816613 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 880486116864 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated17231104 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated17231104 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed17188455411712 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed17639700789594 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed16761457169792 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 _step78 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated10576882 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current252 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 878329423038 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed18068854091776 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 878243619802 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated2873279 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated17230173155328 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated17639803632190 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated16761474209152 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated6654222 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated10576882 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed17230148251648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 " _timestamp1.7857957403737e+09 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed6654220 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current250 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated18134762408960 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed18134604741120 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 904589253632 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated3943334 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated6654222 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed2873029 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 trainer/global_step999 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 880398680064 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121|&M KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated6654222 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated10576882 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed17230148251648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 " _timestamp1.7857957403737e+09 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed6654220 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current250 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated18134762408960 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed18134604741120 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 904589253632 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated3943334 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated6654222 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed2873029 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 trainer/global_step999 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 880398680064 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated17188472451072 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated18068958567936 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed6654220 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed6816361 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed17221432 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed10567212 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 _runtime 453.197054731 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed17221432 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 880398680064 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated17188472451072 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 904456489472 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed17188455411712 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed10567212 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak538 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed3943332 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated6816613 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 880486116864 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated17231104 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated17231104 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed17188455411712 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed17639700789594 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed16761457169792 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 _step78 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated10576882 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current252 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 878329423038 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed18068854091776 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 878243619802 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated2873279 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated17230173155328 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated17639803632190 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated16761474209152 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816Natc0+ _wandbruntime453  _runtime453 w 1loss/SRqqt2/assignment_loss2.674259261239933 .loss/FRt1/detection_loss4.073093729746806 1loss/SRqqt1/detection_loss1.6784740462811756 &loss/total_loss50.869488388679294 $ _timestamp1.785795740375719e+09 _runtime 453.198563298 0loss/FRt1/assignment_loss5.6221791985747105 .loss/FRt2/detection_loss2.874042111894359 0loss/SRqqt2/detection_loss4.291398003508773 trainer/global_step999 0loss/FRt2/assignment_loss5.6728653741919475 1loss/SRqqt1/assignment_loss2.296282558487012 1loss/FBt1/assignment_loss0.18832246263378433 0loss/FBt2/assignment_loss0.6723656396974217 .loss/FBt1/detection_loss5.796786003869706 epoch0 /loss/FBt2/detection_loss15.029419998553665 _step79G5 1loss/SRqqt2/assignment_loss2.674259261239933 .loss/FRt1/detection_loss4.073093729746806 1loss/SRqqt1/detection_loss1.6784740462811756 &loss/total_loss50.869488388679294 $ _timestamp1.785795740375719e+09 _runtime 453.198563298 0loss/FRt1/assignment_loss5.6221791985747105 .loss/FRt2/detection_loss2.874042111894359 0loss/SRqqt2/detection_loss4.291398003508773 trainer/global_step999 0loss/FRt2/assignment_loss5.6728653741919475 1loss/SRqqt1/assignment_loss2.296282558487012 1loss/FBt1/assignment_loss0.18832246263378433 0loss/FBt2/assignment_loss0.6723656396974217 .loss/FBt1/detection_loss5.796786003869706 epoch0 /loss/FBt2/detection_loss15.029419998553665 _step79O|;(: U gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts98.143' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent32.714333333333336 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0uKOY<(:  U* disk./.usagePercent32.343283899485314 disk.drbd1938.out0 disk.rbd1.in0 disk.drbd1673.out0 disk.drbd1938.in0 disk.drbd2041.out0 disk.drbd2078.in0 disk.drbd1931.in0 disk.drbd1552.out0 disk.drbd1289.in0 disk.sda.out 13.10546875 disk.drbd1510.in0! proc.memory.rssMB 11260.46875 disk.drbd1965.out0 disk.drbd1931.out0 disk.drbd2041.in0 disk.dm-0.in0 disk.drbd1226.in0 disk.drbd1828.in0 disk.nvme0n1.in0! disk.nvme0n1.out 11.509765625 disk.rbd1.out 1.9873046875 disk.drbd1289.out0 disk.drbd1285.out0$ memory_percent41.21919274330139 proc.cpu.threads29 disk.drbd1817.out0 disk.drbd1584.out0 disk.drbd2118.in0 disk.drbd1673.in0 disk.drbd1828.out0 disk.drbd1965.in0 disk.drbd2111.out0 network.recv109262 disk.drbd1356.out0 disk.drbd1550.out0 disk.sda1.out0 disk.drbd2111.in0 disk.drbd1552.in0 disk.drbd1285.in0 disk.drbd1510.out0 disk.dm-0.out 11.509765625 cpu14.631476757368812 disk.sda2.in0 disk.drbd1226.out0 disk.drbd1584.in0$ disk./.usageGB578.3307647705078 disk.drbd2118.out0* proc.memory.availableMB38522.58984375 disk.drbd1817.in0 disk.drbd1356.in0 disk.drbd1550.in0 disk.sda1.in0* proc.memory.percent17.182111740112305 disk.sda2.out 13.10546875 disk.rbd0.in0 disk.rbd0.out0 disk.sda.in0 disk.drbd2078.out0 network.sent9446217^-0+ _wandbruntime474  _runtime474G. % _timestamp1.7857957619831882e+09 _runtime 474.806161147 _step80 ' lr-AdamW/pg12.7035023623086093e-05 ' lr-AdamW/pg22.7035023623086093e-05 trainer/global_step1049 _runtime 474.806161147 _step80 ' lr-AdamW/pg12.7035023623086093e-05 ' lr-AdamW/pg22.7035023623086093e-05 trainer/global_step1049 % _timestamp1.7857957619831882e+09P-)0+ _wandbruntime474  _runtime474LDIO UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed18953141995008 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 923602625536 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current281 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 948708615680 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated18953246464000 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed18502980223560 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 921340413212 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated4136247 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed18953141995008 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 _step81 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 923515195904 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed18953141995008 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 923602625536 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current281 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 948708615680 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated18953246464000 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed18502980223560 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 921340413212 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated4136247 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed18953141995008 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 _step81 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 923515195904 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed7156208 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak531 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated4140184 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated18091256 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed18520597869102 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated18046810601472 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated18971292853760 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated17598482825152 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated6986372 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated18090528995328 _runtime 475.188119293 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed6986370 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 924482252288 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed18971188377600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed17598465785792 % _timestamp1.7857957623646526e+09 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 924482252288 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated6986372 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated11104884 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated3016278 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current252 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current254 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed11095214 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated19040271587840 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated11104884 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated18046810601472 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated18091256 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed18081584 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed3016026 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0cSc#M bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated18090528995328 _runtime 475.188119293 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed6986370 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 924482252288 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed18971188377600 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed17598465785792 % _timestamp1.7857957623646526e+09 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 924482252288 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated6986372 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated11104884 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated3016278 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current252 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current254 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed11095214 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated19040271587840 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated11104884 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated18046810601472 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated18091256 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed18081584 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed3016026 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 949742592512 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed11095214 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated18520700711698 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed4140182 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 949609828352 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed7156208 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak531 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated4140184 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated18091256 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed18520597869102 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated18046810601472 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated18971292853760 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated17598482825152 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated6986372 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136Rq0+ _wandbruntime475  _runtime4750p3 0loss/FBt2/assignment_loss0.6645861244243052 .loss/FRt1/detection_loss4.566933308822521 .loss/FRt2/detection_loss3.438584328173323 trainer/global_step1049 /loss/FRt2/assignment_loss5.467570974471721 1loss/SRqqt2/assignment_loss2.673674526781139 %loss/total_loss54.13334050316627 _step83 2loss/SRqqt1/assignment_loss2.3148112856611913 1loss/SRqqt1/detection_loss1.8712816736445033 1loss/SRqqt2/detection_loss3.1825980535238094 /loss/FBt2/detection_loss18.550503856192034 /loss/FRt1/assignment_loss5.643396888954052 1loss/FBt1/assignment_loss0.12637828912497875 .loss/FBt1/detection_loss5.633021193392692 epoch0 % _timestamp1.7857957623664727e+09 _runtime 475.189336x1s /loss/FRt2/assignment_loss5.467570974471721 1loss/SRqqt2/assignment_loss2.673674526781139 %loss/total_loss54.13334050316627 _step83 2loss/SRqqt1/assignment_loss2.3148112856611913 1loss/SRqqt1/detection_loss1.8712816736445033 1loss/SRqqt2/detection_loss3.1825980535238094 /loss/FBt2/detection_loss18.550503856192034 /loss/FRt1/assignment_loss5.643396888954052 1loss/FBt1/assignment_loss0.12637828912497875 .loss/FBt1/detection_loss5.633021193392692 epoch0 % _timestamp1.7857957623664727e+09 _runtime 475.189336 0loss/FBt2/assignment_loss0.6645861244243052 .loss/FRt1/detection_loss4.566933308822521 .loss/FRt2/detection_loss3.438584328173323 trainer/global_step1049S(: ՉU gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts94.601' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.533666666666665 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0M1(:  U disk.drbd2041.out0 disk.drbd1285.out0 disk.drbd1356.out0 disk.drbd2041.in0 disk.drbd1828.in0$ memory_percent41.21896028518677 disk.drbd1817.out0 disk.drbd1550.out0 disk.nvme0n1.in0) disk./.usagePercent32.34328283279611 disk.sda2.in0 disk.drbd2078.out0! proc.memory.rssMB 11260.46875* proc.memory.percent17.182111740112305 disk.sda.in0 disk.drbd1828.out0 disk.drbd1550.in0 disk.drbd1938.in0 disk.rbd0.in0 disk.rbd0.out0 disk.drbd1510.out0 disk.drbd1226.out0 disk.drbd2118.in0 disk.drbd1965.out0 disk.drbd1552.out0 disk.drbd1938.out0 disk.drbd1285.in0 disk.dm-0.in0 disk.drbd1226.in0 disk.drbd1584.in0 disk.rbd1.out 2.11328125 disk.drbd1931.out0 disk.drbd1552.in0 disk.drbd1289.in0 disk.nvme0n1.out 11.71484375 disk.sda1.in0 disk.drbd2118.out0 proc.cpu.threads29 disk.drbd1584.out0 disk.drbd2111.out0 disk.sda2.out 13.515625 cpu14.60294312015351 disk.drbd1673.out0 disk.drbd1356.in0 disk.drbd1817.in0 disk.rbd1.in0 disk.drbd1931.in0 disk.drbd1289.out0) proc.memory.availableMB 38522.7421875 network.sent989683 disk.sda.out 13.515625 disk.sda1.out0 disk.drbd1510.in0 disk.dm-0.out 11.71484375 disk.drbd1965.in0 disk.drbd1673.in0 network.recv112068$ disk./.usageGB578.3307456970215 disk.drbd2078.in0 disk.drbd2111.in0>(: ƩݪU gpu.0.gpu14.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts126.8' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent42.266666666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0q=(:  ƩW disk.drbd1673.out0 disk.drbd1938.out0 disk.sda.in0 disk.sda.out 13.80078125 disk.dm-0.out 11.94140625 disk.drbd2041.out0 disk.drbd1356.in0 disk.drbd1817.out0 disk.rbd1.in0 disk.rbd0.out0 disk.drbd1226.out0 disk.sda2.out 13.80078125* proc.memory.percent17.182111740112305 cpu14.574730039045724 disk.drbd1938.in0 disk.drbd1226.in0 disk.drbd1931.in0 disk.drbd1552.in0 disk.drbd2118.out0 disk.drbd1552.out0 network.sent 1.034695e+06 network.recv114625 disk.drbd1965.in0 disk.drbd1965.out0 disk.drbd1584.in0 disk.sda2.in0 disk.drbd1828.out0 disk.drbd1931.out0 disk.rbd1.out 2.11328125! proc.memory.rssMB 11260.46875 proc.cpu.threads29) disk./.usagePercent32.34329328635031 disk.drbd1550.in0 disk.drbd1289.out0 disk.sda1.in0 disk.drbd1510.in0 disk.dm-0.in0 disk.nvme0n1.in0 disk.drbd1285.out0 disk.drbd1584.out0 disk.drbd2111.out0 disk.drbd2041.in0 disk.drbd2111.in0 disk.drbd2078.in0 disk.nvme0n1.out 11.94140625$ memory_percent41.21902585029602 disk.drbd2078.out0 disk.drbd1356.out0* proc.memory.availableMB38522.69921875$ disk./.usageGB578.3309326171875 disk.drbd1673.in0 disk.drbd1550.out0 disk.drbd2118.in0 disk.drbd1828.in0 disk.drbd1289.in0 disk.rbd0.in0 disk.sda1.out0 disk.drbd1285.in0 disk.drbd1510.out0 disk.drbd1817.in0F,q0+ _wandbruntime497  _runtime497>rt % _timestamp1.7857957845253465e+09 _runtime 497.348308472 _step84 ' lr-AdamW/pg12.8323632947351405e-05 ' lr-AdamW/pg22.8323632947351405e-05 trainer/global_step1099^I _step84 ' lr-AdamW/pg12.8323632947351405e-05 ' lr-AdamW/pg22.8323632947351405e-05 trainer/global_step1099 % _timestamp1.7857957845253465e+09 _runtime 497.348308472T10+ _wandbruntime497  _runtime4971'O QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 _runtime 497.349622928 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated3157777 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current251 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 965143073596 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated7311879 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current253 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated7311879 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated18934228 trainer/global_step1099 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed11612693 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed19383877303068 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated18418751268832 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed19927593776640 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 993940961792 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 967511331328 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed19855476280832 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed11612693 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak541 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed3157526 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed18933652814848 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 967598760960 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 994071635968 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 _step85 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed18924570 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated11622349 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed18887964949504 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak531 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated19855580749824 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 _step85 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed18924570 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated11622349 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed18887964949504 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak531 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated19855580749824 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 _runtime 497.349622928 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated3157777 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current251 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 965143073596 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated7311879 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current253 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated7311879 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated18934228 trainer/global_step1099 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed11612693 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed19383877303068 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated18418751268832 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed19927593776640 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 993940961792 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 967511331328 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed19855476280832 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed11612693 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak541 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed3157526 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176U^0+ _wandbruntime497  _runtime497,{WM R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed19873522663424 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated18905148751872 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak541 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed7318520 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 trainer/global_step1099 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _step86 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated11632886 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed19945703091200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current246 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed3160375 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed7497407 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated11632886 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated3160619 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated18951408 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 994843159552 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated19401597791206 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed18941736 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated18950884835328 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak531 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated19873627139584 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated18951408 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated7497653 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated18905148751872 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed18950859931648 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated7318522 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current246 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed3160375 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed7497407 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated11632886 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated3160619 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated18951408 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 994843159552 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated19401597791206 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed18941736 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated18950884835328 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak531 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated19873627139584 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated18951408 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated7497653 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated18905148751872 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed18950859931648 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated7318522 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated18905148751872 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak541 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed7318520 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 trainer/global_step1099 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _step86 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated11632886 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed19945703091200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 F(:  թU disk.drbd2041.in0 disk.drbd1285.in0 disk.dm-0.out 12.287109375 disk.drbd1584.in0 disk.drbd2118.in0* proc.memory.percent17.182111740112305 disk.drbd1828.out0 disk.rbd0.in0 disk.drbd2111.out0 disk.drbd1510.out0! disk.nvme0n1.out 12.287109375 cpu14.567658005456272 disk.drbd1817.in0 disk.drbd1552.in0 disk.drbd1356.in0 disk.drbd1356.out0 disk.rbd1.out 2.17578125 disk.drbd1931.out0 disk.drbd1938.out0 disk.sda1.in0 disk.drbd2078.out0% memory_percent41.220253705978394 disk.sda2.in0 disk.rbd1.in0 disk.drbd1289.in0 disk.dm-0.in0 network.sent 1.037923e+06$ disk./.usageGB578.3320083618164 disk.drbd1550.out0 disk.drbd2078.in0 disk.sda.in0 disk.drbd1584.out0* proc.memory.availableMB38521.89453125 network.recv115760 disk.nvme0n1.in0) disk./.usagePercent32.34335344762143 disk.drbd1965.out0 proc.cpu.threads29 disk.sda2.out 14.1171875 disk.drbd1828.in0 disk.rbd0.out0 disk.drbd1226.in0 disk.drbd1673.in0 disk.drbd1552.out0 disk.drbd1550.in0 disk.drbd1289.out0 disk.drbd1285.out0 disk.drbd1510.in0 disk.drbd2118.out0 disk.drbd1817.out0 disk.drbd1673.out0 disk.drbd1931.in0 disk.drbd1938.in0! proc.memory.rssMB 11260.46875 disk.drbd1965.in0 disk.sda1.out0 disk.drbd2111.in0 disk.drbd2041.out0 disk.sda.out 14.1171875 disk.drbd1226.out0%ox0+ _wandbruntime519  _runtime519{9 ' lr-AdamW/pg12.9612242271616706e-05 ' lr-AdamW/pg22.9612242271616706e-05 trainer/global_step1149 % _timestamp1.7857958069835172e+09 _runtime 519.806511767 _step88e" ' lr-AdamW/pg22.9612242271616706e-05 trainer/global_step1149 % _timestamp1.7857958069835172e+09 _runtime 519.806511767 _step88 ' lr-AdamW/pg12.9612242271616706e-05XR0+ _wandbruntime519  _runtime519 'O ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 1011594896384 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed12140695 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated19794033558528 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated19746320139264 trainer/global_step1149 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated7644029 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated12150351 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated3301679 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed19794008654848 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated7831626 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated4529947 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current259 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed12140695 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak531 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 1039243609600 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated19746320139264 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated20757915035648 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 1009031537104 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed7644027 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed20757810566656 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated7644029 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 1009117340228 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated20264877225060 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 % _timestamp1.7857958069867547e+09 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated19255759884832 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current261 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _step89 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 1011507466752 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak541 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed20264774382576 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed19255742845472 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated20833277168128 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1011594896384 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated19794380 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed3301420 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 1039104546816 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated12150351 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated3301679 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed19794008654848 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated7831626 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated4529947 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current259 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed12140695 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak531 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 1039243609600 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated19746320139264 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated20757915035648 ZHDeviceStatsMonitor.on_train_batch_start/requestet2d_bytes.small_pool.freed 1009031537104 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed7644027 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed20757810566656 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated7644029 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 1009117340228 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated20264877225060 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 % _timestamp1.7857958069867547e+09 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated19255759884832 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current261 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _step89 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 1011507466752 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak541 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed20264774382576 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed19255742845472 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated20833277168128 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1011594896384 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated19794380 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed3301420 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 1039104546816 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated20775961425408 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed19763469862912 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated19811240675328 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed7650670 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated7838473 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated12160888 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed19801888 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated7650672 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed7650670 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed19811215771648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1012474523136 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 1009909010326 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed19801888 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated20775961425408 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated12160888 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated19272500057152 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated7650672 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated19811560 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated19763486902272 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated4533884 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed19763469862912 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 149279232 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated20851384351744 % _timestamp1.7857958073626137e+09 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed20775856949248 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated3304589 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed20851235072512 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 1012387086336 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current263 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed12151218 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 1012387086336 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated20282494870714 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed7838210 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed4533882 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 124375552 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed19272483017792 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1012474523136 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed12151218 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated19811560 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed3304328 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 _runtime 520.185996014 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1040143676416 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed20282392028118 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated19763486902272 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 trainer/global_step1149*c&M WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed12151218 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 1012387086336 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated20282494870714 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed7838210 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed4533882 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 124375552 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed19272483017792 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1012474523136 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed12151218 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated19811560 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed3304328 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 _runtime 520.185996014 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1040143676416 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed20282392028118 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated19763486902272 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 trainer/global_step1149 _step90 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated20775961425408 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed19763469862912 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated19811240675328 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed7650670 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated7838473 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated12160888 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed19801888 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated7650672 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed7650670 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed19811215771648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1012474523136 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 1009909010326 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed19801888 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated20775961425408 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated12160888 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated19272500057152 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated7650672 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated19811560 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated19763486902272 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated4533884 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed19763469862912 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 149279232 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated20851384351744 % _timestamp1.7857958073626137e+09 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed20775856949248 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated3304589 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed20851235072512 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 1012387086336 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current263ZT0+ _wandbruntime520  _runtime520*sU /loss/FRt1/assignment_loss5.506004220247268 1loss/SRqqt1/assignment_loss2.271897797073637 2loss/SRqqt2/assignment_loss2.6742594694649733 /loss/FBt1/detection_loss4.0851311930457745 _step91 /loss/FRt2/assignment_loss4.991214975580439 .loss/FRt1/detection_loss3.934218116477132 0loss/SRqqt1/detection_loss2.157946386545296 /loss/FBt2/detection_loss25.013537953159986 trainer/global_step1149 0loss/FBt2/assignment_loss0.5165967558725522 .loss/FRt2/detection_loss2.845923085201968 %loss/total_loss57.26812595496234 epoch0 1loss/FBt1/assignment_loss0.43322688066145043 1loss/SRqqt2/detection_loss2.8381691216318696 % _timestamp1.7857958073645911e+09 _runtime 520.18757435Ir trainer/global_step1149 0loss/FBt2/assignment_loss0.5165967558725522 .loss/FRt2/detection_loss2.845923085201968 %loss/total_loss57.26812595496234 epoch0 1loss/FBt1/assignment_loss0.43322688066145043 1loss/SRqqt2/detection_loss2.8381691216318696 % _timestamp1.7857958073645911e+09 _runtime 520.18757435 /loss/FRt1/assignment_loss5.506004220247268 1loss/SRqqt1/assignment_loss2.271897797073637 2loss/SRqqt2/assignment_loss2.6742594694649733 /loss/FBt1/detection_loss4.0851311930457745 _step91 /loss/FRt2/assignment_loss4.991214975580439 .loss/FRt1/detection_loss3.934218116477132 0loss/SRqqt1/detection_loss2.157946386545296 /loss/FBt2/detection_loss25.013537953159986[mL(: U gpu.0.gpu23.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts126.58' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent42.193333333333335 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors03|6(:  U disk.drbd1828.in0 disk.sda.in0 disk.sda.out14.625 disk.dm-0.in0* proc.memory.percent17.182111740112305 disk.drbd2041.in0 disk.drbd1673.out0 disk.drbd1510.in0 disk.drbd1510.out0 disk.drbd1226.out0 network.recv117542 disk.drbd1285.out0 disk.drbd1584.out0 disk.rbd1.in0 disk.rbd1.out 2.17578125 disk.dm-0.out 12.474609375$ memory_percent41.22017025947571 disk.drbd2118.in0 disk.drbd2111.out0 disk.drbd1552.out0 disk.drbd1931.out0 network.sent 1.082778e+06 disk.rbd0.out0! proc.memory.rssMB 11260.46875! disk.nvme0n1.out 12.474609375 disk.drbd1965.in0 disk.drbd2041.out0 disk.drbd1938.out0* proc.memory.availableMB38521.94921875 disk.drbd1356.out0 disk.drbd1550.in0 disk.drbd1938.in0 disk.sda1.out0 disk.sda2.out14.625 disk.drbd1931.in0 disk.drbd1226.in0 disk.drbd1584.in0 disk.drbd1356.in0 disk.drbd1673.in0 disk.drbd2078.in0 disk.drbd2111.in0 disk.drbd1550.out0 disk.drbd1289.in0 disk.sda1.in0 disk.drbd1828.out0 disk.drbd2118.out0 disk.drbd1817.in0 disk.drbd1285.in0 disk.drbd2078.out0* disk./.usagePercent32.343408915460046 disk.drbd1817.out0 disk.drbd1965.out0 disk.drbd1289.out0 disk.rbd0.in0 proc.cpu.threads29$ disk./.usageGB578.3330001831055 disk.drbd1552.in0 cpu14.549879152331519 disk.nvme0n1.in0 disk.sda2.in0m(: ڧU gpu.0.gpu37.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts95.155' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent31.71833333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0o@9(:  U disk.nvme0n1.in0 disk.sda1.in0 disk.drbd1550.out0 disk.dm-0.in0 disk.drbd1673.in0 disk.drbd1510.out0 disk.drbd1226.out0 proc.cpu.threads29 disk.drbd2111.out0 disk.drbd1931.out0 disk.drbd1828.in0 disk.rbd0.in0 disk.drbd1584.in0 disk.drbd2118.in0 disk.drbd1289.in0$ disk./.usageGB578.3331260681152 disk.drbd1356.in0 disk.drbd1817.in0 disk.rbd1.out 2.3017578125 disk.drbd1965.in0& proc.memory.availableMB 38521.5625* proc.memory.percent17.182111740112305 disk.sda2.in0 disk.sda.out 14.953125 disk.sda1.out0 disk.sda.in0 disk.drbd1938.out0 disk.drbd2078.in0 disk.drbd1584.out0 disk.drbd1817.out0! disk.nvme0n1.out 12.716796875 disk.drbd1931.in0 disk.rbd0.out0% memory_percent41.220760345458984 cpu14.525135870891491 network.sent 1.128176e+06 disk.sda2.out 14.953125 disk.drbd1673.out0 disk.dm-0.out 12.716796875 disk.drbd1226.in0 disk.drbd1356.out0 disk.drbd2111.in0 disk.drbd1965.out0 disk.drbd1552.in0 disk.drbd1938.in0 disk.drbd1285.out0 network.recv120992 disk.drbd1550.in0 disk.drbd1552.out0( disk./.usagePercent32.3434159556088 disk.drbd1289.out0 disk.rbd1.in0 disk.drbd2041.in0 disk.drbd1285.in0 disk.drbd2118.out0 disk.drbd1828.out0 disk.drbd2041.out0 disk.drbd1510.in0 disk.drbd2078.out0! proc.memory.rssMB 11260.46875D20+ _wandbruntime542  _runtime542Շ & lr-AdamW/pg13.090085159588201e-05 & lr-AdamW/pg23.090085159588201e-05 trainer/global_step1199 % _timestamp1.7857958292216878e+09 _runtime 542.046711136 _step92H7 v & lr-AdamW/pg13.090085159588201e-05 & lr-AdamW/pg23.090085159588201e-05 trainer/global_step1199 % _timestamp1.7857958292216878e+09 _runtime 542.046711136 _step92\^T0+ _wandbruntime542  _runtime542#O ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed12668697 f%PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated8172985 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed21660144852480 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated3446188 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated7976179 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 % _timestamp1.7857958292223942e+09 _runtime 542.046810029 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed20604641250304 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated12678353 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed4726795 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 1084318797824 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 trainer/global_step1199 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed21660144852480 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated7976179 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated21660249321472 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed21738683292672 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed7976177 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current257 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed20604641250304 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed8172728 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 1084455763456 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated20092768500832 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated20654389398528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated20604658289664 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 1055503602176 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 1055503602176 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated20654532 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated21145774304568 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 1052920000612'O RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak541 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed20092751461472 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed7976177 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated20604658289664 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 1055591031808 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current255 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed21145671462084 _step93 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated12678353 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated4726797 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated21660249321472 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed21660144852480 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated3446188 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated7976179 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 % _timestamp1.7857958292223942e+09 _runtime 542.046810029 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed20604641250304 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated12678353 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed4726795 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 1084318797824 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 trainer/global_step1199 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed21660144852480 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated7976179 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated21660249321472 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed21738683292672 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed7976177 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current257 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed20604641250304 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed8172728 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 1084455763456 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated20092768500832 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated20654389398528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated20604658289664 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 1055503602176 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 1055503602176 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated20654532 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated21145774304568 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 1052920000612 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed12668697 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated8172985 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated20671712 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated20671596515328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 143249920 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed7982820 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated12688890 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed3448776 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed21756787247616 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed20671571611648 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak531 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated20109508673152 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed20621808013312 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated21756955401216 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed20662040 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed20109491633792 _runtime 542.410801295 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed20662040 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated8179770 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated21163391950222 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated20671712 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 1053883277070 trainer/global_step1199 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 1053797473834 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1056470658560 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current262 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed8179508 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed7982820 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _step94 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated12688890 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current260 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated7982822 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 1056383221760 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed21678191235072 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed4730732 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1085358885888 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _step94 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated12688890 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current260 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated7982822 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 1056383221760 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed21678191235072 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed4730732 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1085358885888 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated20671712 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated20671596515328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 143249920 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed7982820 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated12688890 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed3448776 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed21756787247616 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed20671571611648 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak531 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated20109508673152 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed20621808013312 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated21756955401216 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed20662040 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed20109491633792 _runtime 542.410801295 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed20662040 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated8179770 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated21163391950222 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated20671712 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 1053883277070 trainer/global_step1199 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 1053797473834 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1056470658560 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current262 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed8179508 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1^A3t0+ _wandbruntime542  _runtime542JV[ epoch0 % _timestamp1.7857958295893488e+09 0loss/SRqqt1/detection_loss1.379545166850173 .loss/FBt1/detection_loss5.998666482026258 _runtime 542.412176567 1loss/FBt1/assignment_loss0.06870111762734321 .loss/FBt2/detection_loss36.51128927903645 0loss/SRqqt2/assignment_loss2.58670174839473 0loss/FBt2/assignment_loss0.5829905768235525 0loss/SRqqt2/detection_loss3.063972070075498 trainer/global_step1199 _step95 0loss/FRt1/assignment_loss5.0647734486779505 /loss/FRt2/assignment_loss4.911280582691061 1loss/SRqqt1/assignment_loss2.365943739173609 .loss/FRt1/detection_loss3.661690506477689 /loss/FRt2/detection_loss3.7828789012528015 %loss/total_loss69.97843361910711pt 1loss/FBt1/assignment_loss0.06870111762734321 .loss/FBt2/detection_loss36.51128927903645 0loss/SRqqt2/assignment_loss2.58670174839473 0loss/FBt2/assignment_loss0.5829905768235525 0loss/SRqqt2/detection_loss3.063972070075498 trainer/global_step1199 _step95 0loss/FRt1/assignment_loss5.0647734486779505 /loss/FRt2/assignment_loss4.911280582691061 1loss/SRqqt1/assignment_loss2.365943739173609 .loss/FRt1/detection_loss3.661690506477689 /loss/FRt2/detection_loss3.7828789012528015 %loss/total_loss69.97843361910711 epoch0 % _timestamp1.7857958295893488e+09 0loss/SRqqt1/detection_loss1.379545166850173 .loss/FBt1/detection_loss5.998666482026258 _runtime 542.412176567_`ޛ(: U gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp37.0 gpu.0.powerWatts94.173' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.391000000000002 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0B(:  U disk.rbd1.out 2.3017578125 disk.drbd2041.out0 disk.drbd1289.in0 disk.dm-0.out13.02294921875 disk.drbd1584.in0 disk.drbd1673.out0 disk.drbd1938.in0! proc.memory.rssMB 11260.46875 disk.drbd2078.in0 disk.drbd1931.in0 disk.drbd1552.out0 disk.rbd0.out0 disk.drbd1673.in0 disk.sda.out 15.3515625* proc.memory.percent17.182111740112305 disk.drbd2118.in0# disk.nvme0n1.out13.02294921875 disk.drbd1550.out0 disk.rbd0.in0 disk.drbd1965.in0$ memory_percent41.22157692909241 disk.nvme0n1.in0 network.recv122175) disk./.usagePercent32.34346374328515$ disk./.usageGB578.3339805603027 disk.sda1.in0 disk.drbd1285.out0 disk.drbd1510.out0 disk.drbd1931.out0 disk.drbd1356.out0 disk.sda.in0 disk.sda1.out0 disk.sda2.out 15.3515625 disk.drbd1828.in0 disk.drbd1356.in0 disk.sda2.in0 disk.drbd1817.in0 disk.drbd1938.out0* proc.memory.availableMB38521.02734375 disk.drbd1965.out0 disk.drbd1226.out0 disk.drbd2078.out0 disk.drbd1584.out0 disk.drbd2111.in0 disk.drbd1550.in0 network.sent 1.131462e+06 disk.drbd1285.in0 disk.drbd1817.out0 disk.rbd1.in0 disk.dm-0.in0 disk.drbd2041.in0 disk.drbd1828.out0 disk.drbd1289.out0 cpu14.50180154535255 proc.cpu.threads29 disk.drbd1510.in0 disk.drbd1226.in0 disk.drbd2111.out0 disk.drbd1552.in0 disk.drbd2118.out00+ _wandbruntime564  _runtime5643 h $ _timestamp1.785795851626145e+09 _runtime 564.449101322 _step96 & lr-AdamW/pg13.218946092014731e-05 & lr-AdamW/pg23.218946092014731e-05 trainer/global_step1249*'U trainer/global_step1249 $ _timestamp1.785795851626145e+09 _runtime 564.449101322 _step96 & lr-AdamW/pg13.218946092014731e-05 & lr-AdamW/pg23.218946092014731e-05`;o0+ _wandbruntime564  _runtime564C'O HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed8308327 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 1099499737600 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed21462979400704 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 1099499737600 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated22644329886720 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated20929777116832 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated21462996440064 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 1129584648192 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed22562479138304 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 1099587167232 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated13206355 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated22562583607296 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated3589271 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated22026671384076 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 1096894267244 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 trainer/global_step1249 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated21514745238528l['O MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated22026671384076 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 1096894267244 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 trainer/global_step1249 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated21514745238528 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed8308327 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 1099499737600 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed21462979400704 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 1099499737600 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated22644329886720 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated20929777116832 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated21462996440064 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 1129584648192 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed22562479138304 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 1099587167232 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated13206355 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated22562583607296 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated3589271 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360a[0+ _wandbruntime564  _runtime564VM KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated22044289029730 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 1097771740578 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated21531864 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated22580629997056 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed8314970 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed21480146163712 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated21480163203072 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 H@DeviceStatsMonitor.on_train_batch_end/segment.smaibxL!ll_pool.current121 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated21531864 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated8314972 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated20946517289152 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated22662438536704 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 1130355514368 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed8519427 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 1100379357184 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed22662282966016 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current250 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated21531952355328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed22044186187134 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _runtime 564.8202377 _step98 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed3591845 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed21531927451648 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed21522192 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 $ _timestamp1.785795851996832e+09 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed13207222 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _runtime 564.8202377 _step98 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed3591845 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed21531927451648 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed21522192 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 $ _timestamp1.785795851996832e+09 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed13207222 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated21531864 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated8314972 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated20946517289152 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated22662438536704 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 1130355514368 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed8519427 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121b[ȷ0+ _wandbruntime564  _runtime564a߃ 0loss/FBt2/assignment_loss0.4564851120283658 0loss/SRqqt2/detection_loss3.395805409115352 .loss/FBt1/detection_loss5.824647686453413 trainer/global_step1249 2loss/SRqqt1/assignment_loss2.2301693611507174 1loss/SRqqt2/assignment_loss2.416116384977705 1loss/FBt1/assignment_loss0.06689061147849049 0loss/SRqqt1/detection_loss2.093378255023409 $ _timestamp1.785795851999028e+09 _runtime 564.82184535 .loss/FRt2/detection_loss3.858474791569765 &loss/total_loss63.051314244782155 epoch0 /loss/FRt2/assignment_loss4.902136633562487 .loss/FRt1/detection_loss3.702111875188762 /loss/FBt2/detection_loss29.541264281901352 _step99 /loss/FRt1/assignment_loss4.563833842332336P /loss/FRt2/assignment_loss4.902136633562487 .loss/FRt1/detection_loss3.702111875188762 /loss/FBt2/detection_loss29.541264281901352 _step99 /loss/FRt1/assignment_loss4.563833842332336 0loss/FBt2/assignment_loss0.4564851120283658 0loss/SRqqt2/detection_loss3.395805409115352 .loss/FBt1/detection_loss5.824647686453413 trainer/global_step1249 2loss/SRqqt1/assignment_loss2.2301693611507174 1loss/SRqqt2/assignment_loss2.416116384977705 1loss/FBt1/assignment_loss0.06689061147849049 0loss/SRqqt1/detection_loss2.093378255023409 $ _timestamp1.785795851999028e+09 _runtime 564.82184535 .loss/FRt2/detection_loss3.858474791569765 &loss/total_loss63.051314244782155 epoch0c(: ĠU gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts123.39' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent41.13 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0 A(:  U disk.drbd1965.in0 disk.drbd1552.out0 disk.rbd0.in0 disk.drbd1673.in0 disk.drbd2041.out0 disk.drbd1356.in0 disk.drbd1931.in0 disk.drbd1550.in0 disk.drbd1289.out0 disk.drbd1931.out0 disk.drbd1226.out0 disk.drbd2078.out0 disk.rbd1.in0 disk.drbd1552.in0 disk.drbd1938.in0 disk.drbd1285.in0 disk.dm-0.in0 disk.drbd1356.out0 disk.sda2.in0 disk.drbd1510.out0 disk.drbd2111.out0 network.sent 1.177024e+06 disk.sda1.out0% memory_percent41.221678256988525) proc.memory.availableMB 38520.9609375! proc.memory.rssMB 11260.46875* proc.memory.percent17.182111740112305 disk.drbd1938.out0 disk.sda.in0 disk.sda2.out 15.703125 disk.drbd2111.in0) disk./.usagePercent32.34346587666356 disk.drbd1673.out0 disk.drbd1828.out0 disk.sda1.in0# disk.nvme0n1.out13.31005859375 network.recv124968$ disk./.usageGB578.3340187072754 disk.drbd1550.out0 disk.rbd0.out0 disk.drbd2078.in0 disk.nvme0n1.in0 disk.rbd1.out 2.4501953125 disk.drbd1965.out0 disk.drbd1828.in0 disk.drbd1289.in0 disk.drbd2041.in0 disk.drbd1510.in0 disk.drbd1817.in0 disk.drbd1817.out0 disk.drbd2118.in0 disk.drbd1285.out0 disk.drbd1584.out0 disk.drbd2118.out0 proc.cpu.threads29 disk.drbd1226.in0 disk.drbd1584.in0 cpu14.479373095245107 disk.sda.out 15.703125 disk.dm-0.out13.31005859375)!ܐ(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts124.908' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent41.636 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0'F(:  U disk.dm-0.in0 disk.drbd1965.in0 disk.drbd2111.out0 disk.drbd1931.out0 disk.drbd2041.out0 disk.drbd1289.in0 disk.sda.out 16.06640625 disk.drbd1285.in0 disk.dm-0.out14.31494140625 disk.drbd2118.in0 network.recv127382$ disk./.usageGB578.3340377807617 disk.drbd1828.out0 disk.drbd1938.out0 disk.rbd0.in0 disk.drbd1285.out0! proc.memory.rssMB 11260.46875 disk.drbd1226.in0 disk.drbd1673.out0 disk.sda1.out0 disk.drbd1510.in0 disk.rbd0.out0 disk.rbd1.in0 disk.drbd1828.in0 disk.sda.in0 disk.drbd1510.out0 disk.drbd1226.out0 disk.drbd2118.out0* proc.memory.availableMB38520.70703125 disk.drbd1550.in0 disk.drbd1584.in0 network.sent 1.222063e+06 disk.drbd1938.in0 disk.drbd1356.in0 disk.drbd1817.out0 disk.drbd1673.in0 disk.drbd2078.in0 disk.drbd1584.out0* proc.memory.percent17.182111740112305 disk.drbd1965.out0 disk.drbd2041.in0 disk.nvme0n1.in0 disk.drbd1356.out0% memory_percent41.222065687179565 disk.drbd2078.out0# disk.nvme0n1.out14.31494140625 disk.sda2.in0 cpu14.469687000306445) disk./.usagePercent32.34346694335276 disk.drbd1817.in0 disk.drbd1931.in0 disk.drbd1550.out0 disk.drbd2111.in0 disk.drbd1552.in0 disk.sda1.in0 disk.sda2.out 16.06640625 proc.cpu.threads29 disk.rbd1.out 2.4501953125 disk.drbd1552.out0 disk.drbd1289.out0g0+ _wandbruntime586  _runtime586ܚz] _runtime 586.645645997 _step100 & lr-AdamW/pg13.347807024441262e-05 & lr-AdamW/pg23.347807024441262e-05 trainer/global_step1299 % _timestamp1.7857958738213732e+09 & lr-AdamW/pg13.347807024441262e-05 & lr-AdamW/pg23.347807024441262e-05 trainer/global_step1299 % _timestamp1.7857958738213732e+09 _runtime 586.645645997 _step100d!0+ _wandbruntime586  _runtime5867O SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed3731265 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated23549902804480 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated8640479 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated21766785732832 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed8851760 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated3731506 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 1140782730752 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated22375101078528 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated13734357 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated22321334590464 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated13734357 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed23464813424128 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated22321334590464 _runtime 586.645768704 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated22374836 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 1174801725952 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current243 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated5120497 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak541 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed13724701 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.la:3|rge_pool.freed5120495 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed23549747226624 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed8640477 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed13724701 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1143583302656 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated8640479 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 1143583302656 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed8640477 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak531 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 _step101 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0^g'O ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 1143583302656 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed8640477 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak531 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 _step101 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed3731265 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated23549902804480 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated8640479 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated21766785732832 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed8851760 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated3731506 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 1140782730752 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated22375101078528 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated13734357 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated22321334590464 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated13734357 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed23464813424128 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated22321334590464 _runtime 586.645768704 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated22374836 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 1174801725952 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current243 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated5120497 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak541 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed13724701 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed5120495 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed23549747226624 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed8640477 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed13724701 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1143583302656 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated8640479 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392eZ90+ _wandbruntime587  _runtime5876`&M bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated22392308195328 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed22338484314112 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated22392016 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 1175573566976 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 trainer/global_step1299 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed22338484314112 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed21783508865792 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed22382344 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed3734131 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed22925083266642 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated13744894 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1144462929408 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1175706331136 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed13735224 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated23482964282880 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated8647122 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed5124432 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 1144375492608 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 1141574400850 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated13744894 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 1141660204086 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed22382344 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed8647120 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current252 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak541 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated22392016 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated5124434 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed22392283291648 % _timestamp1.7857958741935136e+09 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak531 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed23567856858624 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated22925186109238 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed13735224 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated22338501353472 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated22338501353472 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current254 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated21783525905152 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 _runtime 587.01695642 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated8647122 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated23568014526464 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated23482964282880 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed8858563 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current1217~&M REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated13744894 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 1141660204086 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed22382344 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed8647120 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current252 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak541 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated22392016 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated5124434 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed22392283291648 % _timestamp1.7857958741935136e+09 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak531 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed23567856858624 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated22925186109238 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed13735224 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated22338501353472 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated22338501353472 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current254 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated21783525905152 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 _runtime 587.01695642 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated8647122 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated23568014526464 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated23482964282880 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed8858563 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated22392308195328 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed22338484314112 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated22392016 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 1175573566976 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 trainer/global_step1299 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed22338484314112 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed21783508865792 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed22382344 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed3734131 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed22925083266642 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated13744894 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1144462929408 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1175706331136 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed13735224 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated23482964282880 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated8647122 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed5124432 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 1144375492608 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 1141574400850 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160fж.0+ _wandbruntime587  _runtime587Z 0loss/FRt2/assignment_loss4.6286528293903055 0loss/SRqqt2/detection_loss2.537817611280134 /loss/FBt1/detection_loss7.5612631821151775 %loss/total_loss59.35293289835859 % _timestamp1.7857958741956253e+09 _step103 1loss/SRqqt2/assignment_loss2.065728831089149 .loss/FRt2/detection_loss4.231130716319267 trainer/global_step1299 /loss/FRt1/assignment_loss4.050684909431302 0loss/FBt2/assignme,?nt_loss0.7095735358695189 /loss/FRt1/detection_loss3.3002705923756777 1loss/SRqqt1/detection_loss1.5506568670318268 _runtime 587.018456772 2loss/SRqqt1/assignment_loss2.4157020033046765 0loss/FBt1/assignment_loss0.3293669214651275 .loss/FBt2/detection_loss25.97208489868642 epoch0GA 2loss/SRqqt1/assignment_loss2.4157020033046765 0loss/FBt1/assignment_loss0.3293669214651275 .loss/FBt2/detection_loss25.97208489868642 epoch0 0loss/FRt2/assignment_loss4.6286528293903055 0loss/SRqqt2/detection_loss2.537817611280134 /loss/FBt1/detection_loss7.5612631821151775 %loss/total_loss59.35293289835859 % _timestamp1.7857958741956253e+09 _step103 1loss/SRqqt2/assignment_loss2.065728831089149 .loss/FRt2/detection_loss4.231130716319267 trainer/global_step1299 /loss/FRt1/assignment_loss4.050684909431302 0loss/FBt2/assignment_loss0.7095735358695189 /loss/FRt1/detection_loss3.3002705923756777 1loss/SRqqt1/detection_loss1.5506568670318268 _runtime 587.018456772gd(: U gpu.0.gpu23.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts96.537' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent32.179 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0ZHD(:  U disk.drbd1356.out0 disk.rbd1.in0 disk.sda.out 16.7578125 disk.drbd1285.in0 disk.drbd1226.out0 disk.drbd2078.out0 disk.drbd1817.in0 disk.drbd1584.out0 disk.rbd1.out 2.7001953125 disk.drbd1552.out0) disk./.usagePercent32.34345947652833 disk.rbd0.in0 disk.drbd1510.in0 disk.drbd2078.in0 disk.drbd2118.in0 disk.drbd1828.out0 disk.drbd1510.out0 disk.dm-0.out14.78857421875! proc.memory.rssMB 11260.46875 disk.drbd1356.in0 disk.drbd1550.in0 disk.drbd1289.in0 disk.drbd1965.out0 disk.drbd1226.in0 disk.drbd1965.in0 disk.drbd1938.out0 disk.drbd2041.in0 disk.drbd1289.out0 disk.sda1.in0 disk.drbd1931.out0 disk.drbd1673.in0 disk.rbd0.out0 cpu14.460048615269185 proc.cpu.threads29# disk.nvme0n1.out14.78857421875 network.sent 1.225293e+06 disk.drbd2041.out0 disk.drbd1828.in0 disk.drbd1550.out0 disk.sda.in0* proc.memory.availableMB38520.87890625 disk.drbd2118.out0 disk.drbd1931.in0* proc.memory.percent17.182111740112305 disk.drbd1584.in0 disk.sda2.out 16.7578125 disk.drbd2111.in0 disk.nvme0n1.in0 disk.drbd1673.out0 disk.drbd1552.in0 disk.drbd1938.in0 disk.dm-0.in0$ disk./.usageGB578.3339042663574 disk.drbd2111.out0 disk.sda1.out0% memory_percent41.221803426742554 disk.sda2.in0 disk.drbd1817.out0 disk.drbd1285.out0 network.recv1287010+ _wandbruntime609  _runtime609L8' & lr-AdamW/pg13.476667956867793e-05 & lr-AdamW/pg23.476667956867793e-05 trainer/global_step1349 % _timestamp1.7857958962133503e+09 _runtime 609.038741255 _step104x: trainer/global_step1349 % _timestamp1.7857958962133503e+09 _runtime 609.038741255 _step104 & lr-AdamW/pg13.476667956867793e-05 & lr-AdamW/pg23.476667956867793e-05h'm0+ _wandbruntime609  _runtime609C<'O _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated22603794348832 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated14262359 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed24455281928192 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 1184671194260 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed24367147709952 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated23235456918528 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed23235432014848 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed24367147709952 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 trainer/global_step1349 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated23179672740864 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 1219976393216 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed23225330 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated9191799 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 126479872 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 1187579438080 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated24367252178944 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 1187492008448 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed22603777309472 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed14252703 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current255 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 _runtime 609.039718535 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current253 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed23179655701504 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed23788362700608X})'O OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated8972629 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 1187492008448 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed8972627 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed14252703 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated8972629 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated24367252178944 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 1187492008448 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed22603777309472 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed14252703 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current255 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 _runtime 609.039718535 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current253 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed23179655701504 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed23788362700608 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated22603794348832 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated14262359 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed24455281928192 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 1184671194260 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed24367147709952 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated23235456918528 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed23235432014848 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed24367147709952 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 trainer/global_step1349 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated23179672740864 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 1219976393216 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed23225330 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated9191799 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 126479872 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 1187579438080 IDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated8979272 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed9198248 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated23806083188746 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed8979270 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 1185548667594 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1188459064832 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed14263226 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated23196839503872 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed8979270 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated14272896 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current244 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed5321282 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated8979272 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed23242496 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1220885938176 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 1220748979712 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated14272896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed24473388111360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated23252168 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated5321284 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed22620517481792 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1188459064832 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated23252168 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated9198492 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed3876966 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed23805980346150 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated22620534521152 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 % _timestamp1.7857958965928686e+09 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed14263226 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated3877208 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak`Y 2768240640 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated24385298568704 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 1188371628032 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated24473549973504 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed23242496 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current242 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0Kz&M H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated24385298568704 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 1188371628032 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated24473549973504 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed23242496 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current242 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated8979272 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed9198248 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated23806083188746 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed8979270 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 1185548667594 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1188459064832 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed14263226 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated23196839503872 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed8979270 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated14272896 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current244 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed5321282 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated8979272 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed23242496 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1220885938176 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 1220748979712 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated14272896 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed24473388111360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated23252168 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated5321284 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed22620517481792 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1188459064832 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated23252168 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated9198492 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed3876966 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed23805980346150 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated22620534521152 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 % _timestamp1.7857958965928686e+09 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed14263226 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated3877208 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2jM 0+ _wandbruntime609  _runtime609 1loss/SRqqt1/detection_loss2.3514400015299337 0loss/SRqqt2/detection_loss2.484369408245347 /loss/FBt2/detection_loss15.392909265227596 % _timestamp1.7857958965945623e+09 _runtime 609.417399975 /loss/FRt1/assignment_loss4.250238508336684 1loss/SRqqt1/assignment_loss2.211329774801121 %loss/total_loss50.63830226198387 epoch0 trainer/global_step1349 .loss/FRt2/assignment_loss4.79866903373994 1loss/FBt2/assignment_loss0.29618094761907643 _step107 1loss/FBt1/assignment_loss0.18304525575998318 /loss/FRt1/detection_loss3.7255040652611675 .loss/FRt2/detection_loss3.880555383950831 .loss/FBt1/detection_loss8.934068057834773 2loss/SRqqt2/assignment_loss2.1299925596774125ٔ .loss/FRt2/detection_loss3.880555383950831 .loss/FBt1/detection_loss8.934068057834773 2loss/SRqqt2/assignment_loss2.1299925596774125 1loss/SRqqt1/detection_loss2.3514400015299337 0loss/SRqqt2/detection_loss2.484369408245347 /loss/FBt2/detection_loss15.392909265227596 % _timestamp1.7857958965945623e+09 _runtime 609.417399975 /loss/FRt1/assignment_loss4.250238508336684 1loss/SRqqt1/assignment_loss2.211329774801121 %loss/total_loss50.63830226198387 epoch0 trainer/global_step1349 .loss/FRt2/assignment_loss4.79866903373994 1loss/FBt2/assignment_loss0.29618094761907643 _step107 1loss/FBt1/assignment_loss0.18304525575998318 /loss/FRt1/detection_loss3.7255040652611675k?(: U gpu.0.gpu30.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts122.676' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent40.892 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0S B(:  U network.recv132142 disk.sda1.in0 disk.drbd1552.in0( proc.memory.availableMB 38521.078125$ disk./.usageGB578.3337211608887 disk.drbd1817.out0 disk.drbd1552.out0 disk.drbd1938.in0 disk.sda2.in0 cpu14.450837470401805 disk.rbd0.out0 disk.sda.in0 disk.drbd1584.out0 disk.drbd2118.in0 disk.sda2.out 18.01171875 disk.drbd2111.in0 disk.sda1.out0 disk.dm-0.in0# disk.nvme0n1.out15.77001953125 disk.drbd1550.out0* proc.memory.percent17.182111740112305 disk.drbd1289.in0 disk.sda.out 18.01171875 disk.drbd1828.in0 disk.drbd1356.in0 disk.rbd1.out 2.7001953125 disk.drbd1931.out0 disk.drbd2041.out0) disk./.usagePercent32.34344923631197 disk.drbd1510.in0 disk.drbd1226.out0 disk.drbd1931.in0 disk.drbd2041.in0 disk.drbd1285.in0 disk.dm-0.out15.77001953125 disk.drbd1226.in0 disk.rbd0.in0 disk.drbd2078.out0 disk.drbd2111.out0 disk.drbd1673.in0 network.sent 1.270742e+06 disk.drbd1289.out0 disk.drbd2078.in0 disk.drbd1584.in0# memory_percent41.2214994430542 disk.drbd1285.out0 disk.drbd1965.in0 disk.drbd1550.in0 disk.drbd1938.out0 disk.nvme0n1.in0 disk.drbd1356.out0 disk.drbd1817.in0 disk.drbd1673.out0 disk.drbd1510.out0 disk.drbd2118.out0 disk.drbd1828.out0! proc.memory.rssMB 11260.46875 proc.cpu.threads29 disk.rbd1.in0 disk.drbd1965.out0DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed24085482 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 1264996717568 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 1231488143872 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated25269586464768 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 1231488143872 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated14790361 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed9530523 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 1228559657768 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 _step109 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated14790361 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1231575573504 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current271 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 1265133683200 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed23440785925472 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated9304779 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed9304777 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed14780705 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed4016328 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated24095140 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated25360946441728 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed24669259780116 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 1231575573504 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated9530796 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 1228473854644 trainer/global_step1399 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed24037993851904 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed25269481995776 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed25269481995776 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated5514197 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated23440802964832 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current273 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated4016599 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed24037993851904 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392SO SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed24095787854848 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated24669362622600 _runtime 631.504318047 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated24038010891264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated24095812758528 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed14780705 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed24085482 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 1264996717568 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 1231488143872 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated25269586464768 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 1231488143872 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated14790361 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed9530523 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 1228559657768 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 _step109 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated14790361 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1231575573504 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current271 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytesk.u.small_pool.allocated 1265133683200 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed23440785925472 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated9304779 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed9304777 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed14780705 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed4016328 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated24095140 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated25360946441728 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed24669259780116 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 1231575573504 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated9530796 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 1228473854644 trainer/global_step1399 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed24037993851904 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed25269481995776 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed25269481995776 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated5514197 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated23440802964832 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current273 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated4016599 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed24037993851904 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392m=0+ _wandbruntime631  _runtime631?90|&M C6DeviceStatsMonitor.on_train_batch_end/active.all.freed24102648 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed14791228 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 trainer/global_step1399 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated24055177654272 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated25287632854528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated9537530 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated25379054632960 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed24112994971648 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 % _timestamp1.7857959190602975e+09 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed24055160614912 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 1232367763456 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 _runtime 631.883667543 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated9311422 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed9537270 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated5518134 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed9311420 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed4019138 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated25287632854528 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated24112320 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed24686877425658 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated24112320 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1266034757632 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated14800898 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1232455200256 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current260 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed24055160614912 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 1229437131102D^&M NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed14791228 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 1229351327866 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated24055177654272 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1232455200256 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed5518132 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated24686980268254 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 1265899896320 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed23457526097792 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed9537270 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated5518134 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed9311420 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed4019138 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated25287632854528 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated24112320 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed24686877425658 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated24112320 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1266034757632 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated14800898 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1232455200256 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current260 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed24055160614912 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 1229437131102 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed24102648 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed14791228 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 trainer/global_step1399 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated24055177654272 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated25287632854528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated9537530 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated25379054632960 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed24112994971648 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 % _timestamp1.7857959190602975e+09 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed24055160614912 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 1232367763456 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 _runtime 631.883667543 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated9311422 E(:  ܪ߳U disk.drbd1931.out0 disk.drbd1584.out0 disk.drbd1356.out0* proc.memory.availableMB38521.03515625 cpu14.43358311954871 disk.drbd2041.out0 disk.drbd1510.in0 disk.drbd1965.in0$ memory_percent41.22156500816345 disk.drbd1828.out0 disk.drbd1285.in0 network.sent 1.319034e+06 disk.drbd2041.in0 disk.rbd0.in0 disk.sda.in0 disk.drbd1226.out0 disk.drbd1673.in0 disk.drbd1931.in0 disk.rbd0.out0 disk.sda1.out0 disk.drbd2118.in0 disk.rbd1.in0 disk.drbd1552.in0 disk.sda1.in0 disk.drbd1226.in0 disk.drbd2078.out0 disk.rbd1.out 2.7626953125$ disk./.usageGB578.3338775634766 disk.drbd1550.out0 disk.sda.out 18.98046875 disk.dm-0.in0 disk.nvme0n1.out 16.23828125 network.recv135973 disk.drbd1289.in0 disk.drbd1510.out0 disk.drbd1965.out0 disk.drbd1552.out0 disk.drbd1828.in0 disk.drbd1584.in0 disk.drbd2118.out0 disk.drbd2111.in0 proc.cpu.threads29 disk.dm-0.out 16.23828125 disk.nvme0n1.in0) disk./.usagePercent32.34345798316345 disk.drbd1550.in0 disk.drbd1285.out0 disk.drbd1817.in0! proc.memory.rssMB 11260.46875 disk.drbd1938.in0 disk.drbd1938.out0 disk.sda2.in0 disk.drbd1673.out0 disk.drbd2078.in0 disk.drbd2111.out0 disk.drbd1289.out0* proc.memory.percent17.182111740112305 disk.sda2.out 18.98046875 disk.drbd1817.out0 disk.drbd1356.in0x 0+ _wandbruntime654  _runtime654f?  ' lr-AdamW/pg13.7343898217208536e-05 ' lr-AdamW/pg23.7343898217208536e-05 trainer/global_step1449 % _timestamp1.7857959412484055e+09 _runtime 654.071400199 _step112{ ' lr-AdamW/pg23.7343898217208536e-05 trainer/global_step1449 % _timestamp1.7857959412484055e+09 _runtime 654.071400199 _step112 ' lr-AdamW/pg13.7343898217208536e-05p $0+ _wandbruntime654  _runtime654'DO E8DeviceStatsMonitor.on_train_batch_start/active.all.freed24945634 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated24896349041664 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed24956143694848 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed24896332002304 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated9870194 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated25550259702108 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated15318363 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated24896349041664 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed26171816281600 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated24956168598528 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 1272362318152 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated26171920750592 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 1310151899136 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1275571708928 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed9636927 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated26171920750592 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed26171816281600 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated9636929 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed9636927 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated4159147 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed15308707 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 _runtime 654.072651028 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 1275484279296 % _timestamp1.7857959412493834e+09 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 1275571708928 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated5711047 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated26266453268992 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 1272448121276 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated9636929 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed9869933 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484s޷'O N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed24956143694848 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed24896332002304 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated9870194 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated25550259702108 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated15318363 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated24896349041664 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed26171816281600 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated24956168598528 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 1272362318152 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated26171920750592 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 1310151899136 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1275571708928 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed9636927 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated26171920750592 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed26171816281600 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated9636929 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed9636927 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated4159147 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed15308707 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 _runtime 654.072651028 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 1275484279296 % _timestamp1.7857959412493834e+09 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 1275571708928 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated5711047 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated26266453268992 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 1272448121276 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated9636929 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed9869933 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed24945634 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated24896349041664 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632qs_0+ _wandbruntime654  _runtime6545c&M A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 _step114 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed15319230 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed9876664 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1311184475136 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 1273239791374 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated24294551753152 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated26189967140352 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed24962800 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated15328900 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1276451335680 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 trainer/global_step1449 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated4161915 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed5714982 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 1311055905280 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 1276363898880 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current233 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated24972472 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed4161682 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated24972472 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated5714984 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed9643570 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated24913515804672 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated9876899 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed26284406716928 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed24973350811648 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated26189967140352 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 1276363898880 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed24913498765312 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated25567877347762 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 _runtime 654.447623213 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated24913515804672 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed26189862664192 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed9643570 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current235 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed25567774505166 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed24962800 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 1273325594610 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated9643572 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated26284560190464 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed24294534713792 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 % _timestamp1.7857959416242456e+09 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated9643572 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed25567774505166 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed24962800 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 1273325594610 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated9643572 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated26284560190464 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed24294534713792 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 % _timestamp1.7857959416242456e+09 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated9643572 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed9876664 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1311184475136 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 1273239791374 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated24294551753152 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated26189967140352 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed24962800 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated15328900 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1276451335680 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 trainer/global_step1449 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated4161915 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 QEDeviceStatsMonitor.on_train_batch_end/inactive_split):^.large_pool.freed5714982 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 1311055905280 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 1276363898880 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current233 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated24972472 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed4161682 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated24972472 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated5714984 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed9643570 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated24913515804672 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated9876899 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed26284406716928 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed24973350811648 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated26189967140352 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 1276363898880 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed24913498765312 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated25567877347762 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 _runtime 654.447623213 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated24913515804672 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed26189862664192 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed9643570 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current235 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032rieh0+ _wandbruntime654  _runtime654M 0loss/SRqqt2/detection_loss2.932627814830941 /loss/FBt2/detection_loss15.748997325077656 /loss/FRt2/assignment_loss4.724261830834782 1loss/FBt1/assignment_loss0.12260859163396878 /loss/FBt2/assignment_loss0.690715692093363 /loss/FRt2/detection_loss3.7922569969121147 .loss/FBt1/detection_loss5.802967141565555 &loss/total_loss48.154507057890385 epoch0 /loss/FRt1/assignment_loss4.208794226249059 2loss/SRqqt1/assignment_loss2.1523714268461185 .loss/FRt1/detection_loss4.356752527256806 1loss/SRqqt1/detection_loss1.5178986303437663 _runtime 654.448882083 2loss/SRqqt2/assignment_loss2.1042548542462507 trainer/global_step1449 % _timestamp1.7857959416260304e+09 _step115IY .loss/FBt1/detection_loss5.802967141565555 &loss/total_loss48.154507057890385 epoch0 /loss/FRt1/assignment_loss4.208794226249059 2loss/SRqqt1/assignment_loss2.1523714268461185 .loss/FRt1/detection_loss4.356752527256806 1loss/SRqqt1/detection_loss1.5178986303437663 _runtime 654.448882083 2loss/SRqqt2/assignment_loss2.1042548542462507 trainer/global_step1449 % _timestamp1.7857959416260304e+09 _step115 0loss/SRqqt2/detection_loss2.932627814830941 /loss/FBt2/detection_loss15.748997325077656 /loss/FRt2/assignment_loss4.724261830834782 1loss/FBt1/assignment_loss0.12260859163396878 /loss/FBt2/assignment_loss0.690715692093363 /loss/FRt2/detection_loss3.7922569969121147sh8(: ݋U gpu.0.gpu38.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts109.219' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent36.40633333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0*5(:  V disk.drbd1552.in0 disk.drbd1938.out0* proc.memory.percent17.182111740112305 disk.sda.in0 disk.drbd1510.out0 disk.drbd1226.out0 disk.drbd1584.out0 disk.drbd1673.out0 proc.cpu.threads29 network.recv138198 disk.drbd2118.in0 disk.rbd1.out 2.888671875 disk.drbd1931.in0$ memory_percent41.22203588485718$ disk./.usageGB578.3338584899902 disk.drbd1285.in0 disk.drbd2111.out0 disk.drbd1817.out0 disk.drbd2041.out0 disk.drbd1550.out0 disk.drbd1226.in0 disk.drbd2078.out0 disk.nvme0n1.out16.6875 disk.sda2.in0 disk.drbd1828.out0 disk.sda1.in0 disk.drbd1584.in0 disk.nvme0n1.in0 disk.drbd1828.in0 disk.sda.out 19.33984375 disk.drbd1356.out0 disk.drbd1550.in0 cpu14.423931117993178 disk.drbd1510.in0 disk.dm-0.out16.6875 disk.drbd1289.in0 disk.drbd1673.in0 disk.drbd1938.in0 network.sent 1.363943e+06 disk.drbd2111.in0 disk.rbd1.in0 disk.drbd1552.out0 disk.rbd0.out0 disk.dm-0.in0 disk.sda1.out0 disk.drbd1285.out0 disk.drbd2078.in0 disk.drbd1356.in0 disk.rbd0.in0) proc.memory.availableMB 38520.7265625) disk./.usagePercent32.34345691647424 disk.drbd2118.out0 disk.drbd1817.in0 disk.drbd2041.in0 disk.drbd1289.out0! proc.memory.rssMB 11260.46875 disk.sda2.out 19.33984375 disk.drbd1965.in0 disk.drbd1965.out0 disk.drbd1931.out0Gj(: ؇U gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts119.257' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent39.75233333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0 C(:  ΐV disk.drbd1552.out0 disk.drbd1226.out0* proc.memory.availableMB38520.72265625 disk.drbd1584.in0 disk.drbd2118.out0 disk.drbd2041.out0 disk.drbd1510.out0 disk.rbd1.out 2.888671875 disk.drbd1965.out0 disk.drbd1931.out0 disk.drbd1356.out0 disk.sda2.in0 disk.drbd1289.in0 disk.sda.out 19.7109375 disk.sda2.out 19.7109375 disk.drbd1938.out0 disk.drbd1510.in0 disk.dm-0.out17.25439453125! proc.memory.rssMB 11260.46875 disk.drbd1289.out0 disk.sda.in0 disk.nvme0n1.in0 disk.drbd1828.out0 disk.drbd1285.in0 proc.cpu.threads29 disk.drbd1828.in0# disk.nvme0n1.out17.25439453125 disk.sda1.in0 disk.drbd1673.out0$ disk./.usageGB578.3338394165039 disk.rbd1.in0 disk.drbd1965.in0 disk.drbd1673.in0 disk.drbd1550.in0 disk.drbd1550.out0 disk.drbd1226.in0 network.sent 1.408807e+06) disk./.usagePercent32.34345584978504 disk.drbd1584.out0 disk.drbd1356.in0 disk.drbd1817.out0 disk.drbd1938.in0 disk.drbd1285.out0 disk.drbd2111.in0 disk.drbd2041.in0 disk.rbd0.in0* proc.memory.percent17.182111740112305 disk.drbd2078.in0 disk.drbd2078.out0 disk.drbd1931.in0 disk.rbd0.out0 disk.drbd2111.out0 disk.sda1.out0 disk.drbd1817.in0 disk.dm-0.in0 cpu14.415181200843971 disk.drbd2118.in0% memory_percent41.222041845321655 network.recv140530 disk.drbd1552.in0 40+ _wandbruntime676  _runtime676kg _step116 & lr-AdamW/pg13.863250754147384e-05 & lr-AdamW/pg23.863250754147384e-05 trainer/global_step1499 % _timestamp1.7857959635053437e+09 _runtime 676.328313026|A & lr-AdamW/pg13.863250754147384e-05 & lr-AdamW/pg23.863250754147384e-05 trainer/global_step1499 % _timestamp1.7857959635053437e+09 _runtime 676.328313026 _step116t0+ _wandbruntime676  _runtime676K'O SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 1319567844352 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed4301353 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated9969079 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated4301605 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current252 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 % _timestamp1.7857959635063248e+09 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed27171923765760 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed25816499534848 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed25805786 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed5907895 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 147189248 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed10209248 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 _step117 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated27074255036416 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed15836709 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated10209502 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 1355424230912 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated25754687192064 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 1319480414720 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated25815444 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated27074255036416 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 1316336584784 _runtime 676.329601327 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated9969079 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed9969077 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 trainer/global_step1499 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated25754687192064 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed26431053939132 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1319567844352 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated5907897 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current254 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated25816524438528 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated26431156781616 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 1316250781660 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed25114803157472 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed27074150567424 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed27074150567424 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 122285568 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed25754670152704 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed9969077 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed27074150567424 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 122285568 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed25754670152704 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed9969077 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current252 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 % _timestamp1.7857959635063248e+09 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed27171923765760 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed25816499534848 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed25805786 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed5907895 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 147189248 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed10209248 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 _step117 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated27074255036416 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed15836709 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated10209502 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 1355424230912 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640u0+ _wandbruntime676  _runtime676'} M A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed26448671584674 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed9975720 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 1317128254882 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 1320360034304 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed25822952 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated9975722 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated25833731555328 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated25771853955072 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated25832624 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated25832624 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 _runtime 676.697255747 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 $ _timestamp1.785795963873893e+09 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed5911832 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated15856902 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1356459051520 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated27190190606848 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed25833706651648 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 1356319995904 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed25131543329792 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed25822952 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated15856902 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed15847232 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current265 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1320447471104 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated10216297 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated25131560369152 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 1320360034304 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed10216030 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed25771836915712 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current267 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated4304463 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated27092301426176 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed4304198 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0,&&M UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 _runtime 676.697255747 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 $ _timestamp1.785795963873893e+09 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed5911832 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated15856902 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1356459051520 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated27190190606848 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed25833706651648 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 1356319995904 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed25131543329792 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed25822952 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated15856902 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed15847232 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current265 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1320447471104 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated10216297 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated25131560369152 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 1320360034304 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed10216030 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed25771836915712 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current267 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated4304463 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated27092301426176 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed4304198 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed26448671584674 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed9975720 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 1317128254882 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 1320360034304 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed25822952 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated9975722 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated25833731555328 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated25771853955072 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated25832624 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated25832624 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121v0+ _wandbruntime676  _runtime676e" 0loss/FRt2/assignment_loss4.7209630332342005 1loss/FBt1/assignment_loss0.17709603848690617 .loss/FRt2/detection_loss3.919373269124729 1loss/SRqqt2/detection_loss2.4730176712691256 .loss/FBt1/detection_loss7.851157053531184 2loss/SRqqt2/assignment_loss1.9083070396322064 .loss/FRt1/detection_loss3.795291296592573 %loss/total_loss48.21334684365657 /loss/FRt1/assignment_loss4.190315714696559 0loss/FBt2/assignment_loss0.6176672865135165 1loss/SRqqt1/detection_loss2.2491194087955426 epoch0 $ _timestamp1.785795963875596e+09 _runtime 676.698435634 _step119 1loss/SRqqt1/assignment_loss1.914253025182656 /loss/FBt2/detection_loss14.396786006597372 trainer/global_step14992W /loss/FRt1/assignment_loss4.190315714696559 0loss/FBt2/assignment_loss0.6176672865135165 1loss/SRqqt1/detection_loss2.2491194087955426 epoch0 $ _timestamp1.785795963875596e+09 _runtime 676.698435634 _step119 1loss/SRqqt1/assignment_loss1.914253025182656 /loss/FBt2/detection_loss14.396786006597372 trainer/global_step1499 0loss/FRt2/assignment_loss4.7209630332342005 1loss/FBt1/assignment_loss0.17709603848690617 .loss/FRt2/detection_loss3.919373269124729 1loss/SRqqt2/detection_loss2.4730176712691256 .loss/FBt1/detection_loss7.851157053531184 2loss/SRqqt2/assignment_loss1.9083070396322064 .loss/FRt1/detection_loss3.795291296592573 %loss/total_loss48.21334684365657wZ(: жU gpu.0.gpu11.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts118.998' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent39.666000000000004 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0p?(:  U disk.drbd1285.in0* proc.memory.percent17.182111740112305 cpu14.407119040761351 disk.drbd2078.in0 disk.drbd1289.in0 disk.drbd1552.out0 disk.rbd0.in0 disk.drbd1510.in0 disk.sda.out 20.015625 disk.drbd1584.in0 disk.sda2.out 20.015625 disk.drbd2111.out0 disk.drbd1673.out0* disk./.usagePercent32.343459049852655 disk.dm-0.out17.48486328125 disk.drbd1356.in0 disk.drbd1828.in0 disk.drbd1828.out0 network.recv141667 disk.drbd2118.out0 disk.drbd1817.in0 disk.drbd1552.in0 disk.drbd1550.out0 disk.drbd1584.out0 disk.drbd1356.out0 disk.drbd1550.in0( proc.memory.availableMB 38520.671875 disk.drbd2111.in0 disk.rbd1.in0 disk.drbd2041.out0 disk.sda1.out0 network.sent 1.411846e+06 disk.drbd2041.in0! proc.memory.rssMB 11260.46875 proc.cpu.threads29 disk.dm-0.in0 disk.nvme0n1.in0 disk.drbd1289.out0 disk.drbd1938.out0 disk.drbd1285.out0 disk.drbd1965.out0# disk.nvme0n1.out17.48486328125 disk.drbd1817.out0 disk.drbd1931.in0 disk.drbd1226.in0 disk.drbd1226.out0 disk.drbd1510.out0 disk.drbd1931.out0$ memory_percent41.22211933135986 disk.drbd2118.in0 disk.drbd1673.in0 disk.sda1.in0 disk.drbd1965.in0 disk.sda.in0 disk.drbd1938.in0$ disk./.usageGB578.3338966369629 disk.drbd2078.out0 disk.sda2.in0 disk.rbd1.out 2.951171875 disk.rbd0.out0r0+ _wandbruntime699  _runtime699J ' lr-AdamW/pg23.9921116865739145e-05 trainer/global_step1549 % _timestamp1.7857959863153462e+09 _runtime 699.138325987 _step120 ' lr-AdamW/pg13.9921116865739145e-05  _step120 ' lr-AdamW/pg13.9921116865739145e-05 ' lr-AdamW/pg23.9921116865739145e-05 trainer/global_step1549 % _timestamp1.7857959863153462e+09 _runtime 699.138325987x<0+ _wandbruntime699  _runtime699'O PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated10301229 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated27312053861124 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 1363476550144 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed25951811773472 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated27976589322240 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated26675596 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed10549504 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed26613008303104 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed16364711 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed4444759 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 1360139245168 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed26665938 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated6104747 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current252 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated25951828812832 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated4445011 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 _step121 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current254 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated27976589322240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed27976484853248 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated16374367 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated10301229 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed10301227 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1363563979776 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed27311951018640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed10301227 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated16374367 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated10549758 $ _timestamp1.785795986316318e+09 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed27976484853248 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 1360225048292 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated26613025342464 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated26676880278528 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121.O XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed26613008303104 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed16364711 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed4444759 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 1360139245168 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed26665938 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated6104747 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current252 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated25951828812832 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.sma"#ll_pool.allocated4445011 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 _step121 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current254 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated27976589322240 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed27976484853248 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated16374367 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated10301229 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed10301227 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1363563979776 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed27311951018640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed10301227 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated16374367 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated10549758 $ _timestamp1.785795986316318e+09 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed27976484853248 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 1360225048292 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated26613025342464 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated26676880278528 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated10301229 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated27312053861124 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 1363476550144 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed25951811773472 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated27976589322240 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated26675596 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed10549504y#㛇0+ _wandbruntime699  _runtime699`&M E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed10307870 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed26694062491648 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed25968551945792 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated26692776 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 1364356169728 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed4447525 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 % _timestamp1.7857959866944478e+09 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1364443606528 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1364443606528 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 _step122 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed28095546308096 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed10556207 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 1361016718390 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated4447793 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current268 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1401618677760 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 1361102521626 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated16384904 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated26692776 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated27329671506778 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed10307870 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed27329568664182 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 trainer/global_step1549 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed26630175066112 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated28095706073088 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed26683104 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed16375234 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed28095546308096 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed10556207 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 1361016718390 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated4447793 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current268 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1401618677760 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 1361102521626 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated16384904 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated26692776 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated27329671506778 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed10307870 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed27329568664182 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 trainer/global_step1549 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed26630175066112 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated28095706073088 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed26683104 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed16375234 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed25968551945792 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated26692776 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 1364356169728 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed4447525 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 % _timestamp1.7857959866944478e+09 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1364443606528zw0+ _wandbruntime699  _runtime6994 0loss/FRt1/assignment_loss3.8327603453681585 /loss/FRt2/detection_loss3.4505897981425124 %loss/total_loss44.98956120513948 _runtime 699.519032922 0loss/FBt1/assignment_loss0.0919588377158082 0loss/FBt2/assignment_loss0.4821174234384671 0loss/SRqqt1/detection_loss1.815379331933449 /loss/SRqqt2/detection_loss3.03324494354843 /loss/FBt2/detection_loss13.435948148864554 _step123 -loss/FRt2/assignment_loss4.7020930952496 .loss/FBt1/detection_loss5.147798572905654 epoch0 2loss/SRqqt1/assignment_loss2.1150781251071544 2loss/SRqqt2/assignment_loss1.9517867634905146 .loss/FRt1/detection_loss4.930805819375174 trainer/global_step1549 % _timestamp1.7857959866961854e+096 -loss/FRt2/assignment_loss4.7020930952496 .loss/FBt1/detection_loss5.147798572905654 epoch0 2loss/SRqqt1/assignment_loss2.1150781251071544 2loss/SRqqt2/assignment_loss1.9517867634905146 .loss/FRt1/detection_loss4.930805819375174 trainer/global_step1549 % _timestamp1.7857959866961854e+09 0loss/FRt1/assignment_loss3.8327603453681585 /loss/FRt2/detection_loss3.4505897981425124 %loss/total_loss44.98956120513948 _runtime 699.519032922 0loss/FBt1/assignment_loss0.0919588377158082 0loss/FBt2/assignment_loss0.4821174234384671 0loss/SRqqt1/detection_loss1.815379331933449 /loss/SRqqt2/detection_loss3.03324494354843 /loss/FBt2/detection_loss13.435948148864554 _step123{(: ѲU gpu.0.gpu0.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts117.407' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent39.135666666666665 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0<=(:  U! proc.memory.rssMB 11260.46875 disk.rbd1.in0 disk.drbd1931.out0 disk.drbd1552.in0 disk.rbd0.in0 disk.drbd1584.out0 disk.drbd1817.in0 disk.drbd1938.in0* proc.memory.availableMB38520.87109375 disk.drbd1817.out0 disk.sda.in0 disk.drbd2078.in0 disk.drbd1584.in0 disk.drbd1356.in0 disk.drbd1673.in0 disk.drbd1828.in0 disk.drbd1938.out0 disk.dm-0.in0 disk.drbd1226.out0 disk.nvme0n1.out 17.74609375 disk.drbd2041.in0 disk.sda1.out0 network.recv143931 disk.rbd1.out 2.951171875 disk.drbd1673.out0$ disk./.usageGB578.3339157104492 disk.drbd2118.in0 disk.drbd1356.out0 disk.drbd1965.in0 disk.sda1.in0 disk.drbd1285.out0 disk.drbd1510.in0 disk.drbd1552.out0 disk.drbd1289.in0 disk.drbd1510.out0 disk.nvme0n1.in0 disk.drbd1965.out0 disk.rbd0.out0 disk.drbd1285.in0 disk.drbd1226.in0* proc.memory.percent17.182111740112305 disk.drbd1828.out0* disk./.usagePercent32.343460116541856 disk.drbd1550.out0 disk.drbd1289.out0 disk.dm-0.out 17.74609375 disk.drbd2118.out0 disk.drbd2111.out0 disk.drbd1931.in0 disk.drbd2041.out0 disk.drbd1550.in0 cpu14.399043890047976 proc.cpu.threads29 disk.drbd2111.in0 disk.sda.out 20.3359375$ memory_percent41.22181534767151 network.sent 1.456722e+06 disk.drbd2078.out0 disk.sda2.in0 disk.sda2.out 20.3359375pXn(: U gpu.0.gpu38.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625LQ") gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts117.108' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent39.036 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0.8(:  U disk.drbd2078.in0) disk./.usagePercent32.34345584978504$ disk./.usageGB578.3338394165039 disk.sda2.in0 disk.drbd1673.in0 disk.drbd2078.out0 disk.rbd1.out 3.107421875 disk.drbd1828.out0 disk.drbd1285.out0 disk.drbd1510.out0! proc.memory.rssMB 11260.46875 disk.drbd1965.out0 disk.drbd2111.in0 disk.drbd1931.out0 disk.drbd2041.out0 disk.rbd0.out0 disk.drbd1285.in0* proc.memory.availableMB38520.62109375 disk.sda1.in0 disk.drbd1938.in0 disk.dm-0.in0 disk.nvme0n1.in0 disk.drbd1289.out0 disk.rbd0.in0 disk.sda.in0 disk.drbd1289.in0 disk.drbd1226.in0 disk.nvme0n1.out17.96875 disk.drbd1510.in0 disk.drbd1584.in0 disk.drbd2118.in0 disk.drbd1965.in0 network.recv146096 disk.drbd2041.in0 disk.drbd2118.out0 disk.drbd1828.in0* proc.memory.percent17.182111740112305 disk.drbd1550.out0 disk.drbd1356.out0$ memory_percent41.22219681739807 disk.drbd1552.in0 disk.drbd1817.in0 disk.dm-0.out17.96875 network.sent 1.501652e+06 disk.drbd2111.out0 disk.drbd1931.in0 disk.drbd1552.out0 disk.sda.out 20.66796875 disk.drbd1226.out0 disk.drbd1584.out0 disk.drbd1356.in0 disk.sda2.out 20.66796875 disk.rbd1.in0 disk.drbd1673.out0 disk.sda1.out0 cpu14.393499317994317 proc.cpu.threads29 disk.drbd1817.out0 disk.drbd1550.in0 disk.drbd1938.out0;20+ _wandbruntime721  _runtime721" & lr-AdamW/pg24.120972619000445e-05 trainer/global_step1599 % _timestamp1.7857960087194343e+09 _runtime 721.542394039 _step124 & lr-AdamW/pg14.120972619000445e-056 & lr-AdamW/pg24.120972619000445e-05 trainer/global_step1599 % _timestamp1.7857960087194343e+09 _runtime 721.542394039 _step124 & lr-AdamW/pg14.120972619000445e-05|4iU0+ _wandbruntime721  _runtime721k'O XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed27471346453504 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 1445907179520 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed6301595 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated10890246 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current244 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 1445774408192 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated16902369 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated28983143298048 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated27537236118528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _runtime 721.543630867 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated27471363492864 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 1404113511800 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed28878819139072 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed4588407 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed27471346453504 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 1407472685568 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated27535748 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed28878819139072 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated27471363492864 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed10890002 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1407560115200 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed10633377 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 $ _timestamp1.785796008720389e+09 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed16892713 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 1407560115200 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed28192848098148 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 1407472685568 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 1576750082/'O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated28878923608064 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated4588649 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 trainer/global_step1599 _step125 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed16892713 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 1407560115200 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed28192848098148 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 1407472685568 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed27471346453504 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 1445907179520 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed6301595 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated10890246 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current244 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 1445774408192 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated16902369 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated28983143298048 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated27537236118528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _runtime 721.543630867 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated27471363492864 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 1404113511800 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed28878819139072 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed4588407 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed27471346453504 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 1407472685568 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated27535748 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed28878819139072 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated27471363492864 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed10890002 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1407560115200 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed10633377 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 $ _timestamp1.785796008720389e+09 IDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed16903236 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 1446679241728 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1408439741952 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated16912906 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed27543256 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed29001097573376 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed28896865521664 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed16903236 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated10640022 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated6305534 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated27554443235328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated26805577601152 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 _runtime 721.919485237 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated16912906 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated27488530255872 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated28896969997824 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 trainer/global_step1599 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed27543256 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed10640020 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated27552928 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current244 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed10896792 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1408439741952 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated4591504 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed27554418331648 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 % _timestamp1.7857960090961614e+09 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated27552928 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed27488513216512 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032z&M E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 % _timestamp1.7857960090961614e+09 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated27552928 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed27488513216512 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 _step126 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed16903236 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 1446679241728 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1408439741952 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated16912906 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed27543256 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed29001097573376 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed28896865521664 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed16903236 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated10640022 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated6305534 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated27554443235328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated26805577601152 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 _runtime 721.919485237 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated16912906 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated27488530255872 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated28896969997824 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 trainer/global_step1599 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed27543256 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed10640020 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated27552928 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current244 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed10896792 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1408439741952 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated4591504 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed27554418331648~+T0+ _wandbruntime721  _runtime721VsJ .loss/FRt2/detection_loss3.730388417051119 .loss/FBt1/detection_loss7.396992347588684 .loss/FBt2/detection_loss24.27334817945957 trainer/global_step1599 % _timestamp1.7857960090978656e+09 _step127 /loss/FRt1/assignment_loss4.148090141160147 2loss/SRqqt2/assignment_loss2.0831597805023194 1loss/FBt1/assignment_loss0.10907067860642672 0loss/FBt2/assignment_loss0.3328922466374934 %loss/total_loss57.88269267305618 /loss/FRt2/assignment_loss4.818011926202213 2loss/SRqqt1/assignment_loss1.6889851857957088 .loss/FRt1/detection_loss3.686833781145868 _runtime 721.920701544 1loss/SRqqt1/detection_loss2.3979923332428883 1loss/SRqqt2/detection_loss3.2169276556637336 epoch0mY /loss/FRt1/assignment_loss4.148090141160147 2loss/SRqqt2/assignment_loss2.0831597805023194 1loss/FBt1/assignment_loss0.10907067860642672 0loss/FBt2/assignment_loss0.3328922466374934 %loss/total_loss57.88269267305618 /loss/FRt2/assignment_loss4.818011926202213 2loss/SRqqt1/assignment_loss1.6889851857957088 .loss/FRt1/detection_loss3.686833781145868 _runtime 721.920701544 1loss/SRqqt1/detection_loss2.3979923332428883 1loss/SRqqt2/detection_loss3.2169276556637336 epoch0 .loss/FRt2/detection_loss3.730388417051119 .loss/FBt1/detection_loss7.396992347588684 .loss/FBt2/detection_loss24.27334817945957 trainer/global_step1599 % _timestamp1.7857960090978656e+09 _step127((: ϹU gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts122.749' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent40.916333333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0cC(:  V disk.sda1.out0 disk.dm-0.out18.39599609375 disk.drbd2078.in0 disk.drbd1817.in0 disk.rbd1.out 3.107421875 disk.drbd1931.out0 disk.nvme0n1.in0# disk.nvme0n1.out18.39599609375 disk.drbd2111.out0 disk.drbd2041.in0 proc.cpu.threads29 disk.drbd1289.in0 disk.sda.out 21.01953125 disk.sda1.in0 disk.drbd1285.in0 disk.drbd1938.out0 disk.drbd1550.out0 disk.drbd1226.out0 disk.drbd1550.in0 disk.drbd1356.in0 disk.drbd1965.in0$ disk./.usageGB578.3337821960449 disk.drbd1289.out0 disk.sda.in0 disk.drbd2118.in0 disk.drbd2118.out0 disk.drbd1673.out0! proc.memory.rssMB 11260.46875 disk.dm-0.in0 disk.drbd1965.out0) proc.memory.availableMB 38520.8046875* proc.memory.percent17.182111740112305 cpu14.379267937169226 disk.drbd1584.out0 disk.drbd1673.in0 disk.drbd1828.in0 disk.drbd1828.out0 disk.drbd1226.in0 disk.drbd1552.out0 disk.drbd1584.in0 disk.drbd1817.out0 disk.rbd0.out0 disk.drbd1510.out0 disk.sda2.in0 disk.rbd1.in0 network.sent 1.504624e+06 network.recv146972 disk.drbd1552.in0 disk.drbd1931.in0 disk.drbd2041.out0$ memory_percent41.22191667556763) disk./.usagePercent32.34345264971743 disk.drbd1938.in0 disk.rbd0.in0 disk.drbd1356.out0 disk.drbd2111.in0 disk.drbd1285.out0 disk.drbd1510.in0 disk.drbd2078.out0 disk.sda2.out 21.01953125~[0+ _wandbruntime743  _runtime743hR! ' lr-AdamW/pg14.2498335514269754e-05 ' lr-AdamW/pg24.2498335514269754e-05 trainer/global_step1649 % _timestamp1.7857960306707869e+09 _runtime 743.493768219 _step128?6 ' lr-AdamW/pg24.2498335514269754e-05 trainer/global_step1649 % _timestamp1.7857960306707869e+09 _runtime 743.493768219 _step128 ' lr-AdamW/pg14.2498335514269754e-05^_L0+ _wandbruntime743  _runtime743'O L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed28329684603904 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed28329684603904 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated29781257893888 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated27625846044832 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed28386242 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated28397591958528 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 trainer/global_step1649 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed27625829005472 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated10965529 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated4731804 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed29781153424896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1451556250624 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current253 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed4731551 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed10965527 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121Yv*(!O JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated28395900 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated17430371 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated11230251 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated29781257893888 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated27625846044832 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed28386242 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated28397591958528 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 trainer/global_step1649 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed27625829005472 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated10965529 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated4731804 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed29781153424896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1451556250624 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current253 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed4731551 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed10965527 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed28329684603904 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed28329684603904 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1452435877376 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed29799199807488 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated28413080 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 1491825031680 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 1452348440576 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed10972170 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed10972170 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated28414799075328 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated29906761065472 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 1448793645406 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 1452348440576 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed28346851366912 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated29091465665794 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 trainer/global_step1649 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed29906599203328 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1491961990144 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated29799304283648 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed28414774171648 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated10972172 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated28346868406272 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated10972172 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated27642586217152 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed28346851366912 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current237 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated11236951 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1452435877376 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 1448879448642 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated28346868406272 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 $ _timestamp1.785796031039033e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed17431238 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed11236714 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed29091362823198 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated28413080 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 1491825031680 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 1452348440576 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed10972170 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed10972170 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated28414799075328 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated29906761065472 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 1448793645406 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 1452348440576 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed28346851366912 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated29091465665794 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 trainer/global_step1649 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed29906599203328 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1491961990144 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated29799304283648 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed28414774171648 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated10972172 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated28346868406272 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated10972172 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated27642586217152 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed28346851366912 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current237 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated11236951 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1452435877376 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 1448879448642 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated28346868406272 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 $ _timestamp1.785796031039033e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed17431238 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed11236714 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed29091362823198 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251%0+ _wandbruntime743  _runtime7434 1loss/FBt1/assignment_loss0.22269745335222801 .loss/FBt1/detection_loss4.408543810503201 $ _timestamp1.785796031040921e+09 _runtime 743.863769442 2loss/SRqqt2/assignment_loss2.1948703156577216 0loss/FBt2/assignment_loss0.3514122677205402 1loss/SRqqt1/detection_loss1.6540806889883815 trainer/global_step1649 /loss/FRt1/assignment_loss4.181003085402555 .loss/FRt2/detection_loss4.391166117663185 epoch0 .loss/FRt1/detection_loss3.620525148718856 1loss/SRqqt2/detection_loss3.3254918935501743 /loss/FBt2/detection_loss10.440181796908291 %loss/total_loss41.26895662467384 _step131 0loss/FRt2/assignment_loss4.7406661212444305 1loss/SRqqt1/assignment_loss1.738317924964277{H2 1loss/FBt1/assignment_loss0.22269745335222801 .loss/FBt1/detection_loss4.408543810503201 $ _timestamp1.785796031040921e+09 _runtime 743.863769442 2loss/SRqqt2/assignment_loss2.1948703156577216 0loss/FBt2/assignment_loss0.3514122677205402 1loss/SRqqt1/detection_loss1.6540806889883815 trainer/global_step1649 /loss/FRt1/assignment_loss4.181003085402555 .loss/FRt2/detection_loss4.391166117663185 epoch0 .loss/FRt1/detection_loss3.620525148718856 1loss/SRqqt2/detection_loss3.3254918935501743 /loss/FBt2/detection_loss10.440181796908291 %loss/total_loss41.26895662467384 _step131 0loss/FRt2/assignment_loss4.7406661212444305 1loss/SRqqt1/assignment_loss1.738317924964277 (: ū붃U gpu.0.gpu32.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts115.57' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent38.52333333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0W(:  ūU disk.drbd1510.out0 disk.drbd2078.in0 disk.nvme0n1.in 2.47265625 disk.sda2.in0 disk.drbd1965.out0 disk.drbd2041.in0$ disk./.usageGB578.3338394165039 disk.drbd1550.out0 disk.sda2.out 21.35546875 cpu14.364507626783062 proc.cpu.threads29 disk.drbd1510.in0 disk.drbd1226.out0 disk.drbd1584.in0 disk.drbd1965.in0 network.sent 1.54982e+06 disk.drbd1285.in0 disk.drbd1285.out0 disk.rbd1.out 3.1396484375* proc.memory.percent17.182111740112305 disk.dm-0.out18.61474609375 disk.drbd1289.in0 disk.drbd1289.out0 disk.sda1.out0 disk.drbd1584.out0 disk.drbd1817.in0 disk.drbd1817.out0 disk.drbd1673.in0 disk.drbd1931.out0 disk.drbd2041.out0% memory_percent41.221994161605835 disk.drbd1938.out0) disk./.usagePercent32.34345584978504# disk.nvme0n1.out18.61474609375 disk.rbd1.in0* proc.memory.availableMB38520.75390625 disk.drbd1552.in0 disk.rbd0.out0 disk.dm-0.in 2.47265625 disk.drbd1673.out0 disk.drbd1552.out0 disk.drbd1828.in0 disk.drbd1828.out0 disk.drbd1356.in0 disk.drbd2111.out0 network.recv149193 disk.sda1.in0 disk.drbd1550.in0 disk.rbd0.in0 disk.sda.in0 disk.drbd2111.in0! proc.memory.rssMB 11260.46875 disk.drbd1226.in0 disk.drbd2118.in0 disk.drbd1938.in0 disk.drbd2078.out0 disk.drbd1356.out0 disk.sda.out 21.35546875 disk.drbd2118.out0 disk.drbd1931.in0H(:  ԫU disk.drbd1550.in0 disk.drbd1938.out0 disk.sda.in0 disk.drbd1584.in0 disk.drbd1550.out0 disk.drbd1285.in0 disk.drbd1226.out0 disk.drbd1356.in0$ disk./.usageGB578.3338584899902 disk.drbd1965.in0 disk.drbd1673.in0 disk.drbd1510.out0 disk.drbd2118.in0 disk.drbd2118.out0 disk.drbd1817.in0 disk.rbd1.out 3.1396484375 disk.drbd1552.in0 disk.drbd1285.out0) disk./.usagePercent32.34345691647424 disk.drbd1552.out0 disk.drbd1828.out0 disk.drbd2078.out0 disk.drbd2111.out0 disk.drbd2041.out0 disk.drbd1289.out0 disk.dm-0.in 2.47265625 disk.drbd1226.in0 disk.drbd1965.out0* proc.memory.percent17.182111740112305 network.recv150753 disk.drbd2041.in0 disk.drbd1510.in0 cpu14.349949457546241 disk.drbd1673.out0 disk.rbd0.in0# disk.nvme0n1.out18.88525390625 disk.drbd2111.in0 disk.sda1.out0 disk.drbd1931.in0 disk.drbd1931.out0 disk.drbd1828.in0 disk.drbd1289.in0 disk.nvme0n1.in 2.47265625 disk.sda2.in0 disk.sda2.out21.6875 disk.drbd1938.in0 disk.sda1.in0 disk.drbd1584.out0 proc.cpu.threads29 disk.drbd1356.out0 disk.drbd1817.out0$ memory_percent41.22200012207031$ proc.memory.availableMB38520.75! proc.memory.rssMB 11260.46875 disk.rbd0.out0 disk.dm-0.out18.88525390625 disk.drbd2078.in0 disk.rbd1.in0 network.sent 1.59461e+06 disk.sda.out21.6875(: ԫU gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts95.348' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.782666666666664 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0Z 0+ _wandbruntime766  _runtime766dC & lr-AdamW/pg14.378694483853506e-05 & lr-AdamW/pg24.378694483853506e-05 trainer/global_step1699 $ _timestamp1.785796053178116e+09 _runtime 766.001047658 _step132 trainer/global_step1699 $ _timestamp1.785796053178116e+09 _runtime 766.001047658 _step132 & lr-AdamW/pg14.378694483853506e-05 & lr-AdamW/pg24.378694483853506e-050+ _wandbruntime766  _runtime766ys$KO ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 1495464956416 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed11569144 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed28462837621472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed30683487710720 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated29256052 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated30794193125376 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed30683487710720 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 _runtime 766.002212005 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 1491890438816 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 1491804635692 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed17948717 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated30683592179712 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed4873849 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated17958373 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed6695295 trainer/global_step1699 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1495552386048 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed30794037547520 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated29188039793664 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed11297677 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 _step133 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 1536245326848 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed29954642257164 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated11569380 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated30683592179712 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated11297679 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed29188022754304 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 % _timestamp1.7857960531790304e+09 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated17958373 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed29257922894848 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 1495552386048 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed17948717 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 1495464956416 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated4874083 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360Ѿ'O H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 _step133 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 1536245326848 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed29954642257164 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated11569380 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated30683592179712 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated11297679 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed29188022754304 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 % _timestamp1.7857960531790304e+09 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated17958373 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed29257922894848 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 1495552386048 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed17948717 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 1495464956416 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated4874083 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 1495464956416 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed11569144 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed28462837621472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed30683487710720 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated29256052 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated30794193125376 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed30683487710720 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 _runtime 766.002212005 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 1491890438816 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 1491804635692 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed17948717 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated30683592179712 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed4873849 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated17958373 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed6695295 trainer/global_step1699 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1495552386048 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed30794037547520 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated29188039793664 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed11297677 I8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated29273232 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated17968910 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed29263560 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated29205206556672 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated28479594833152 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1496432012800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated11304322 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed17959240 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 trainer/global_step1699 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1496432012800 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated29275154915328 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed11304320 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated11576185 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed4876707 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed6699232 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 _step134 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated17968910 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 1492767912150 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated11304322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed17959240 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed30701534093312 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed30812158451712 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated30701638569472 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated29205206556672 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated6699234 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed29263560 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 1537028440064 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 1496344576000 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current244 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 1492682108914 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed11575939 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 1496344576000 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed29205189517312 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed28479577793792 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated4876951vNM J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1496432012800 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated29275154915328 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed11304320 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated11576185 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed4876707 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed6699232 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 _step134 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated17968910 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 1492767912150 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated11304322 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed17959240 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed30701534093312 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed30812158451712 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated30701638569472 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated29205206556672 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated6699234 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed29263560 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 1537028440064 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 1496344576000 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current244 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 1492682108914 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed11575939 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 1496344576000 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed29205189517312 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed28479577793792 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated4876951 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated29273232 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated17968910 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed29263560 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated29205206556672 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated28479594833152 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1496432012800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated11304322 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed17959240 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 trainer/global_step1699(0+ _wandbruntime766  _runtime766K /loss/FRt1/assignment_loss4.112325754510351 0loss/FBt2/assignment_loss0.5070226360324093 .loss/FBt1/detection_loss6.258492580900112 %loss/total_loss50.58528914568177 % _timestamp1.7857960535524356e+09 .loss/FRt2/detection_loss4.430867159928101 0loss/SRqqt1/detection_loss2.500028363995155 _step135 2loss/SRqqt2/assignment_loss2.0067573469877242 .loss/FRt1/detection_loss3.714139971388392 1loss/SRqqt2/detection_loss2.8694520958932115 /loss/FRt2/assignment_loss4.573465495869733 2loss/SRqqt1/assignment_loss1.6713788706809283 0loss/FBt1/assignment_loss0.4147611799970975 /loss/FBt2/detection_loss17.526597689498555 epoch0 trainer/global_step1699 _runtime 766.3752648895!+5 0loss/SRqqt1/detection_loss2.500028363995155 _step135 2loss/SRqqt2/assignment_loss2.0067573469877242 .loss/FRt1/detection_loss3.714139971388392 1loss/SRqqt2/detection_loss2.8694520958932115 /loss/FRt2/assignment_loss4.573465495869733 2loss/SRqqt1/assignment_loss1.6713788706809283 0loss/FBt1/assignment_loss0.4147611799970975 /loss/FBt2/detection_loss17.526597689498555 epoch0 trainer/global_step1699 _runtime 766.375264889 /loss/FRt1/assignment_loss4.112325754510351 0loss/FBt2/assignment_loss0.5070226360324093 .loss/FBt1/detection_loss6.258492580900112 %loss/total_loss50.58528914568177 % _timestamp1.7857960535524356e+09 .loss/FRt2/detection_loss4.430867159928101&rtS(:  U disk.drbd1285.out0 disk.drbd1938.out0 disk.sda1.in0 disk.sda1.out0 disk.drbd1510.in0) disk./.usagePercent32.34345584978504 disk.drbd1828.out0 disk.drbd1289.in0 disk.drbd1550.in0 disk.dm-0.in 2.47265625 disk.drbd1226.in0 disk.drbd1356.in0 disk.drbd2118.in0 disk.drbd1673.in0$ memory_percent41.22209548950195 disk.drbd1510.out0 network.recv153313$ disk./.usageGB578.3338394165039 disk.drbd1938.in0 disk.drbd2078.in0# disk.nvme0n1.out19.18603515625 disk.drbd1550.out0 disk.sda.in0 disk.drbd1584.in0 disk.drbd1356.out0 disk.drbd1817.out0* proc.memory.percent17.182111740112305 disk.drbd1931.in0 disk.rbd0.out0& proc.memory.availableMB 38520.6875 disk.drbd1828.in0 proc.cpu.threads29 disk.drbd1285.in0 disk.drbd2078.out0 disk.nvme0n1.in 2.47265625 disk.drbd1552.in0 disk.drbd1965.in0 disk.dm-0.out19.18603515625 disk.sda2.in0 disk.drbd1817.in0 disk.drbd1673.out0 network.sent 1.639508e+06 disk.drbd1552.out0 disk.rbd0.in0 disk.sda2.out 22.12890625 disk.drbd2111.out0 disk.rbd1.in0 disk.drbd1965.out0 disk.drbd2041.in0 disk.drbd2041.out0 disk.drbd1226.out0 disk.drbd1289.out0 disk.rbd1.out 3.2958984375! proc.memory.rssMB 11260.46875 disk.drbd1931.out0 disk.sda.out 22.12890625 disk.drbd2111.in0 disk.drbd1584.out0 disk.drbd2118.out0 cpu14.335329317894919/(: ̂U gpu.0.gpu32.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts94.442' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent31.48066666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0`0+ _wandbruntime788  _runtime788,< & lr-AdamW/pg24.507555416280036e-05 trainer/global_step1749 $ _timestamp1.785796075309405e+09 _runtime 788.13236174 _step136 & lr-AdamW/pg14.507555416280036e-05- _step136 & lr-AdamW/pg14.507555416280036e-05 & lr-AdamW/pg24.507555416280036e-05 trainer/global_step1749 $ _timestamp1.785796075309405e+09 _runtime 788.13236174M"0+ _wandbruntime788  _runtime788K'O TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed31585821996544 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated30116204 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 1539461091840 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated11629829 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed11629827 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed18476719 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed30106546 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 1581317955072 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated18486375 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current249 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed11909008 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed6892145 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated31699760656384 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed30118278734848 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 1581457017856 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed30835539336672 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated6892147 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 _runtime 788.1336312 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed11629827 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated11909257 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated5017110 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated18486375 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current247 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated30835642179156 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed31585821996544 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated30116204 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 1539461091840 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated11629829 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed11629827 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed18476719 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed30106546 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 1581317955072 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated18486375 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current249 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed11909008 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed6892145 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated31699760656384 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed30118278734848 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 1581457017856 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed30835539336672 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated6892147 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 _runtime 788.1336312 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed11629827 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated11909257 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated5017110 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated18486375 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current247 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated30835642179156 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed31585821996544 IDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated30133384 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated18496912 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed11636470 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed6896082 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 166056448 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed30063527667712 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated31603972855296 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 1536570572422 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed30135485851648 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed30853156982214 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 % _timestamp1.7857960756866236e+09 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated6896084 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated5019889 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1582361516544 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated31603972855296 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed18487242 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed31603868379136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed11636470 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated11915973 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated18496912 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 1540340711424 _step138 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed30123712 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated30063544707072 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated31717872271872 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed29316586409792 trainer/global_step1749 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated11636472 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current238 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current240 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed11915733 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed5019651 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136#m&M TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 % _timestamp1.7857960756866236e+09 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated6896084 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated5019889 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1582361516544 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated31603972855296 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed18487242 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed31603868379136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed11636470 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated11915973 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated18496912 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 1540340711424 _step138 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed30123712 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated30063544707072 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated31717872271872 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed29316586409792 trainer/global_step1749 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated11636472 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current238 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current240 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed11915733 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed5019651 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated30853259824810 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 141152768 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated30133384 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated18496912 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed11636470 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed6896082 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 166056448 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed30063527667712 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated31603972855296 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 1536570572422 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed30135485851648 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed30853156982214 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ODeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 1583544656896 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 1579581562708 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 _runtime 810.273002143 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 1626641183744 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated30904716094464 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated30136871892832 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 1583457227264 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated5159425 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed19004721 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed30966698 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed32488156282368 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed11961977 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated32605300662272 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed31716436416180 trainer/global_step1799 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated11961979 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated12248422 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed30136854853472 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 1579667365832 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated32488260751360 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 126479872 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated19014377 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated32488260751360 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed32488156282368 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 _step141 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed32605149278720 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated30978659478528 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1583544656896 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 1583457227264 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated31716539258664 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current250 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed5159177 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0kKO WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed11961977 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1583544656896 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 1583457227264 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated31716539258664 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current250 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed5159177 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed"Ls0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated30976356 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed19004721 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 1583544656896 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 1579581562708 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 _runtime 810.273002143 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 1626641183744 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated30904716094464 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated30136871892832 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 1583457227264 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated5159425 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed19004721 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed30966698 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed32488156282368 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed11961977 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated32605300662272 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed31716436416180 trainer/global_step1799 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated11961979 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated12248422 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed30136854853472 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 1579667365832 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated32488260751360 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 126479872 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated19014377 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated32488260751360 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed32488156282368 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 _step141 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed32605149278720 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated30978659478528 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated30993536 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 1580544839166 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated32506307141120 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated30995866595328 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current255 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 1584336846848 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed30983864 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current253 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed32623256502272 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1627545477632 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 1627414810624 _step142 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed19015244 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed11968620 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated12255232 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated19024914 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated7092934 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated5162298 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated30993536 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed30921865818112 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed12254977 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 1584336846848 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed5162045 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 1580459035930&M E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 1580544839166 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated32506307141120 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated30995866595328 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current255 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 1584336846848 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed30983864 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current253 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed32623256502272 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1627545477632 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 1627414810624 _step142 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed19015244 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed11968620 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated12255232 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated19024914 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated7092934 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated5162298 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated30993536 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed30921865818112 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed12254977 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 1584336846848 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed5162045 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 1580459035930 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated309935360+ _wandbruntime810  _runtime810Ey[ 0loss/SRqqt1/detection_loss2.072968762446871 /loss/FBt2/detection_loss16.627791774272918 epoch0 trainer/global_step1799 % _timestamp1.7857960978215756e+09 0loss/FRt1/assignment_loss4.3535726530985395 2loss/SRqqt2/assignment_loss2.0317297177808777 .loss/FBt1/detection_loss7.277861325338238 2loss/SRqqt1/assignment_loss1.8423041129249267 .loss/FRt1/detection_loss3.520306288518689 &loss/total_loss49.668068804470536 _runtime 810.64442547 /loss/FRt2/assignment_loss4.716453275562804 0loss/FBt1/assignment_loss0.0929941582758147 0loss/SRqqt2/detection_loss2.467270101765108 _step143 .loss/FBt2/assignment_loss0.87313252389431 .loss/FRt2/detection_loss3.791684110591441\%  2loss/SRqqt2/assignment_loss2.0317297177808777 .loss/FBt1/detection_loss7.277861325338238 2loss/SRqqt1/assignment_loss1.8423041129249267 .loss/FRt1/detection_loss3.520306288518689 &loss/total_loss49.668068804470536 _runtime 810.64442547 /loss/FRt2/assignment_loss4.716453275562804 0loss/FBt1/assignment_loss0.0929941582758147 0loss/SRqqt2/detection_loss2.467270101765108 _step143 .loss/FBt2/assignment_loss0.87313252389431 .loss/FRt2/detection_loss3.791684110591441 0loss/SRqqt1/detection_loss2.072968762446871 /loss/FBt2/detection_loss16.627791774272918 epoch0 trainer/global_step1799 % _timestamp1.7857960978215756e+09 0loss/FRt1/assignment_loss4.3535726530985395˰F(: U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts115.939' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent38.64633333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0HګP(:  ɪU disk.drbd2041.in0 disk.drbd1938.in0 disk.dm-0.in 2.47265625 cpu14.296809112495703 disk.drbd2041.out0 disk.drbd1828.in0 disk.drbd1285.in0 disk.drbd1226.in0 disk.sda2.out 23.3515625 disk.drbd1552.out0 disk.drbd1550.in0 disk.drbd1289.in0 disk.sda1.out0 disk.drbd2078.in0$ memory_percent41.22222661972046 disk.drbd1550.out0 network.sent 1.687288e+06) disk./.usagePercent32.34345712981208 disk.rbd0.in0 disk.drbd2111.in0 disk.rbd1.in0 disk.drbd1931.out0 disk.sda1.in0 disk.drbd1510.out0* proc.memory.percent17.182111740112305 disk.drbd1285.out0 disk.drbd1226.out0 disk.sda2.in0 disk.drbd2118.in0 disk.drbd2111.out0 disk.drbd1938.out0! proc.memory.rssMB 11260.46875 proc.cpu.threads29 disk.rbd0.out0 disk.rbd1.out3.328125 disk.drbd1289.out0 disk.drbd1510.in0 disk.drbd1356.out0 disk.drbd1817.out0 disk.drbd1965.in0 disk.drbd1965.out0 disk.dm-0.out19.81103515625 disk.drbd2078.out0 disk.drbd1584.out0 disk.nvme0n1.in 2.47265625 disk.drbd1931.in0 disk.drbd1552.in0 disk.sda.out 23.3515625) proc.memory.availableMB 38520.6015625 disk.drbd2118.out0 disk.drbd1817.in0 network.recv155735$ disk./.usageGB578.3338623046875# disk.nvme0n1.out19.81103515625 disk.drbd1673.out0 disk.sda.in0 disk.drbd1356.in0 disk.drbd1673.in0 disk.drbd1828.out0 disk.drbd1584.in0Bi0+ _wandbruntime832  _runtime832#O _runtime 832.324298026 _step144 & lr-AdamW/pg14.765277281133098e-05 & lr-AdamW/pg24.765277281133098e-05 trainer/global_step1849 $ _timestamp1.785796119501339e+0946 & lr-AdamW/pg14.765277281133098e-05 & lr-AdamW/pg24.765277281133098e-05 trainer/global_step1849 $ _timestamp1.785796119501339e+09 _runtime 832.324298026 _step14450+ _wandbruntime832  _runtime832\mO E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated33390595037184 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed33510690116096 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed31838990414848 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated12588586 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed12588323 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated5302739 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 1671836666880 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 1671699701248 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed19532723 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed31763037205504 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated32597436338172 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated12294129 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed5302478 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed31826850 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated31839015318528 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 % _timestamp1.7857961195022824e+09 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current261 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated33510851985408 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated30973880508832 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1627540792320 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 trainer/global_step1849 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed7285845 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated31763054244864 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current263 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed31763037205504 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed33390490568192 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated31763054244864 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed33390490568192 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 1627540792320 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated31836508 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated19542379 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed33390490568192 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 1627540792320 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated31836508 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated19542379 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed33510690116096 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed31838990414848 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated12588586 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed12588323 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated5302739 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 1671836666880 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 1671699701248 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed19532723 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed31763037205504 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated32597436338172 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated12294129 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed5302478 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed31826850 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated31839015318528 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 % _timestamp1.7857961195022824e+09 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current261 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated33510851985408 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated30973880508832 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1627540792320 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 trainer/global_step1849 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed72858450+ _wandbruntime832  _runtime832U4&M ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 1672606943232 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 $ _timestamp1.785796119871187e+09 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed7289782 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated5305629 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed31856197531648 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 1624347499438 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed31844016 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated19552916 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated19552916 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed31844016 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated7289784 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed32614951141230 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated33408641426944 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current257 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 trainer/global_step1849 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed12595156 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime 832.69453376 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed12300770 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated31780221007872 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed19543246 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated12595413 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated33408641426944 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1628420419072 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1628420419072 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 _step146 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated31853688 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated12300772 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated31780221007872 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed5305374 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated31856222435328 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 1628332982272 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated32615053983826 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated30990620681152 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed30990603641792 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated31853688 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current255 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated33528957948416 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1672735513088 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320IE M REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated31780221007872 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed19543246 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated12595413 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated33408641426944 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1628420419072 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1628420419072 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 _step146 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated31853688 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated12300772 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated31780221007872 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed5305374 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated31856222435328 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 1628332982272 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 C9DeviceStatsMonitor.on_train_babitch_end/allocation.all.peak11887 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated32615053983826 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated30990620681152 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed30990603641792 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated31853688 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current255 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated33528957948416 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1672735513088 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 1672606943232 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 $ _timestamp1.785796119871187e+09 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed7289782 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated5305629 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed31856197531648 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 1624347499438 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed31844016 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated19552916 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated19552916 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed31844016 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated7289784 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed32614951141230 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated33408641426944 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current257 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 trainer/global_step1849 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed12595156 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime 832.69453376 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed12300770 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936+0+ _wandbruntime832  _runtime832DB /loss/FRt2/detection_loss3.6005438481183614 1loss/SRqqt2/detection_loss2.2706503998838463 &loss/total_loss43.160877042007726 _runtime 832.695977107 1loss/SRqqt1/assignment_loss1.979009337453957 .loss/FBt2/detection_loss13.30308030660336 epoch0 trainer/global_step1849 $ _timestamp1.785796119873158e+09 /loss/FRt2/assignment_loss4.468732292511884 1loss/FBt1/assignment_loss0.16275618971023406 0loss/FBt2/assignment_loss0.4955797548508809 1loss/SRqqt1/detection_loss2.6525741565898797 .loss/FBt1/detection_loss4.550235276528631 _step147 /loss/FRt1/assignment_loss4.064945899738985 2loss/SRqqt2/assignment_loss2.0213726605989235 .loss/FRt1/detection_loss3.5913969194187849۵ /loss/FRt1/assignment_loss4.064945899738985 2loss/SRqqt2/assignment_loss2.0213726605989235 .loss/FRt1/detection_loss3.591396919418784 /loss/FRt2/detection_loss3.6005438481183614 1loss/SRqqt2/detection_loss2.2706503998838463 &loss/total_loss43.160877042007726 _runtime 832.695977107 1loss/SRqqt1/assignment_loss1.979009337453957 .loss/FBt2/detection_loss13.30308030660336 epoch0 trainer/global_step1849 $ _timestamp1.785796119873158e+09 /loss/FRt2/assignment_loss4.468732292511884 1loss/FBt1/assignment_loss0.16275618971023406 0loss/FBt2/assignment_loss0.4955797548508809 1loss/SRqqt1/detection_loss2.6525741565898797 .loss/FBt1/detection_loss4.550235276528631 _step147(: U gpu.0.gpu23.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts125.416' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent41.80533333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0U8RwP(:  U disk.drbd2041.in0 disk.drbd1828.out0 disk.drbd1938.out0 disk.drbd2111.in0! proc.memory.rssMB 11260.46875 disk.drbd1817.out0 disk.drbd1938.in0 disk.drbd1550.in0 network.recv158212 disk.rbd1.in0 disk.rbd0.out0 disk.sda.in0 disk.dm-0.out20.06884765625 disk.drbd1226.in0 disk.drbd2078.out0 disk.rbd1.out3.484375 disk.drbd2041.out0 disk.drbd1552.out0 disk.drbd1828.in0 disk.rbd0.in0 disk.drbd2078.in0 disk.drbd2118.out0# disk.nvme0n1.out20.06884765625 disk.drbd1356.in0 disk.drbd2111.out0 disk.drbd1817.in0 disk.drbd1550.out0 disk.drbd1289.in0 network.sent 1.734799e+06 disk.drbd1965.out0 disk.drbd1552.in0 cpu14.29328673856502 disk.drbd1285.in0 disk.drbd1226.out0 disk.drbd1584.in0 disk.sda1.in0 disk.sda1.out0 disk.drbd1673.in0 disk.drbd1931.in0 disk.drbd1356.out0$ disk./.usageGB578.3314895629883 disk.drbd1965.in0 disk.drbd1931.out0 disk.nvme0n1.in 2.47265625 disk.sda2.in0 disk.sda.out 23.7578125 disk.drbd1285.out0 disk.drbd1510.in0 disk.drbd1584.out0 disk.drbd1289.out0$ memory_percent41.22326970100403 disk.sda2.out 23.7578125* proc.memory.availableMB38519.91796875) disk./.usagePercent32.34332443367507 disk.drbd1510.out0 disk.drbd2118.in0* proc.memory.percent17.182111740112305 disk.drbd1673.out0 disk.dm-0.in 2.47265625 proc.cpu.threads29|0+ _wandbruntime854  _runtime8549 trainer/global_step1899 % _timestamp1.7857961420223336e+09 _runtime 854.845298849 _step148 ' lr-AdamW/pg14.8941382135596276e-05 ' lr-AdamW/pg24.8941382135596276e-05U ' lr-AdamW/pg24.8941382135596276e-05 trainer/global_step1899 % _timestamp1.7857961420223336e+09 _runtime 854.845298849 _step148 ' lr-AdamW/pg14.8941382135596276e-05fc0+ _wandbruntime854  _runtime854QZ'O ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 1671536927744 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated34292929323008 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated32621392395264 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed32621375355904 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 1667358489724 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed12626277 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 1716999854592 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 trainer/global_step1899 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 1671449498112 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated32699371158528 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed31810872085472 _step149 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed12928135 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed12626277 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed34292824854016 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed20060725 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated20070381 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 1671449498112 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current235 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated5445673 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed32687002 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated34292929323008 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated20070381 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 % _timestamp1.7857961420238569e+09 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated34416371013120 _runtime 854.847076717 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed34416211240960 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed34292824854016 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated33478333417680 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated7482697 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 1716864986112 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed32621375355904 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed5445440 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1671536927744 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated12626279 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed32699346254848 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed34292824854016 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated33478333417680 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated7482697 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 1716864986112 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed32621375355904 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed5445440 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1671536927744 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated12626279 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed32699346254848 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated34292929323008 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated32621392395264 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed32621375355904 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 1667358489724 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed12626277 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 1716999854592 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 trainer/global_step1899 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 1671449498112 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated32699371158528 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed31810872085472 _step149 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed12928135 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed12626277 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed34292824854016 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed20060725 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated20070381 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 1671449498112 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current235 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated5445673 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0T(: U gpu.0.gpu41.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts129.04' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent43.01333333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0Ɩ(:  U disk.drbd1356.out0 disk.sda2.out 24.21484375 disk.drbd2111.out0 disk.drbd1817.out0$ memory_percent41.22360944747925 disk.rbd0.in0 network.sent 1.800927e+06 disk.drbd2111.in0 disk.drbd1552.in0 disk.sda1.in0 disk.rbd1.in0 disk.drbd1828.in0 disk.drbd1584.out0# disk.nvme0n1.out20.29052734375 disk.rbd1.out3.484375 disk.drbd1931.in0 disk.drbd2041.in0 disk.drbd2041.out0 disk.drbd1226.in0 disk.drbd1550.in0 disk.rbd0.out0 disk.sda.out 24.21484375 disk.dm-0.in 2.47265625 disk.drbd1965.in0 disk.drbd1289.out0 disk.drbd1938.in0 disk.drbd2078.out0 disk.drbd1673.in0 disk.drbd1673.out0 disk.drbd1226.out0 disk.drbd1356.in0* proc.memory.percent17.182111740112305 disk.dm-0.out20.29052734375 disk.drbd2078.in0T0) proc.memory.availableMB 38519.6953125! proc.memory.rssMB 11260.46875 proc.cpu.threads29 disk.drbd1550.out0 disk.drbd2118.out0 disk.drbd1931.out0 cpu14.289586915797557 disk.drbd1285.out0 disk.drbd1510.out0 disk.drbd1817.in0 network.recv162493( disk./.usagePercent32.3433214469453 disk.drbd2118.in0 disk.nvme0n1.in 2.47265625 disk.drbd1828.out0 disk.drbd1289.in0 disk.drbd1285.in0 disk.drbd1552.out0 disk.sda.in0$ disk./.usageGB578.3314361572266 disk.sda1.out0 disk.sda2.in0 disk.drbd1965.out0 disk.drbd1938.out0 disk.drbd1510.in0 disk.drbd1584.in0xj[0+ _wandbruntime855  _runtime855W&M KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed33495848220738 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated32713840 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated32716578275328 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1717897857024 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed20071248 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed32716553371648 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated7486634 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed34434320561664 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 1717767190016 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed31827612257792 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 % _timestamp1.7857961423939989e+09 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated32713840 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 1668235962946 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 1672329117696 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed20071248 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed12934870 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed32638542118912 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed34310871236608 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated12632922 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 1672329117696 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed32638542118912 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current252 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 _step150 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1672416554496 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1672416554496 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 1668321766182 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated20080918 _runtime 855.217326285 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated32638559158272 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated5448488 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated20080918 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current250 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated34434476132352 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed32704168 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated33495951063334 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed12632920 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed5448238 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 trainer/global_step1899 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated12632922 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed32704168 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed7486632 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated31827629297152 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated34310975712768 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated34310975712768]1&M NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated20080918 _runtime 855.217326285 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated32638559158272 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated5448488 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated20080918 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current250 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated34434476132352 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed32704168 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated33495951063334 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed12632920 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed5448238 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 trainer/global_step1899 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated12632922 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed32704168 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed7486632 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated31827629297152 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated34310975712768 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated34310975712768 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed33495848220738 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated32713840 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated32716578275328 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1717897857024 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed20071248 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed32716553371648 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated7486634 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed34434320561664 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 1717767190016 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed31827612257792 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 % _timestamp1.7857961423939989e+09 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated32713840 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 1668235962946 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 1672329117696 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed20071248 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed12934870 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed32638542118912 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed34310871236608 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated12632922 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 1672329117696 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed32638542118912 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current252 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 _step150 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1672416554496 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1672416554496 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 1668321766182 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated00+ _wandbruntime855  _runtime8553߯ 0loss/FBt2/assignment_loss0.7780751477354594 .loss/FRt1/detection_loss3.342815479604147 /loss/FRt1/assignment_loss4.228450172690935 /loss/FRt2/assignment_loss4.394246773547437 0loss/FBt1/assignment_loss0.2155383623299153 /loss/FRt2/detection_loss3.5782590493200774 %loss/total_loss49.51038259381765 epoch0 trainer/global_step1899 2loss/SRqqt1/assignment_loss1.5847347186336034 1loss/SRqqt2/assignment_loss1.956003207600478 1loss/SRqqt1/detection_loss1.9851123144487457 .loss/FBt1/detection_loss6.226925806449349 .loss/FBt2/detection_loss19.49761574467023 % _timestamp1.7857961423956375e+09 _runtime 855.218449564 _step151 1loss/SRqqt2/detection_loss1.7226058167872742f3 /loss/FRt1/assignment_loss4.228450172690935 /loss/FRt2/assignment_loss4.394246773547437 0loss/FBt1/assignment_loss0.2155383623299153 /loss/FRt2/detection_loss3.5782590493200774 %loss/total_loss49.51038259381765 epoch0 trainer/global_step1899 2loss/SRqqt1/assignment_loss1.5847347186336034 1loss/SRqqt2/assignment_loss1.956003207600478 1loss/SRqqt1/detection_loss1.9851123144487457 .loss/FBt1/detection_loss6.226925806449349 .loss/FBt2/detection_loss19.49761574467023 % _timestamp1.7857961423956375e+09 _runtime 855.218449564 _step151 1loss/SRqqt2/detection_loss1.7226058167872742 0loss/FBt2/assignment_loss0.7780751477354594 .loss/FRt1/detection_loss3.342815479604147{c(: 侐U gpu.0.gpu41.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts95.058' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.686000000000003 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0 M(:  U disk.drbd1828.out0 disk.drbd1550.in0 disk.drbd1289.out0 disk.rbd1.in0 disk.drbd2041.out0 disk.rbd0.in0 disk.rbd0.out0 disk.sda.out 24.796875 disk.drbd1356.out0 disk.drbd1673.in0 disk.drbd1931.in0 disk.drbd1828.in0 disk.drbd1226.in0 disk.drbd2118.out0 disk.drbd1550.out0 disk.drbd1817.in0 disk.rbd1.out 3.6328125* proc.memory.availableMB38519.37109375 disk.drbd1938.out0 disk.sda1.out0 disk.dm-0.in 2.47265625 disk.drbd1584.in0 disk.sda2.in0) disk./.usagePercent32.34343387598743 disk.drbd2078.out0! disk.nvme0n1.out 20.517578125* proc.memory.percent17.182111740112305 disk.sda2.out 24.796875 disk.sda.in0 disk.drbd2118.in0 disk.drbd1965.out0 disk.drbd1226.out0 disk.drbd2111.out0 disk.drbd1817.out0! proc.memory.rssMB 11260.46875 disk.drbd1931.out0 disk.drbd1285.in0 disk.drbd1285.out0 disk.drbd1510.out0 disk.drbd2111.in0$ disk./.usageGB578.3334465026855 disk.drbd2041.in0 disk.sda1.in0 disk.dm-0.out 20.517578125 disk.drbd2078.in0 network.sent 1.845839e+06 disk.drbd1289.in0 disk.drbd1938.in0 disk.drbd1965.in0 cpu14.286425576296606 disk.drbd1673.out0 disk.drbd1552.out0 disk.drbd1510.in0% memory_percent41.224104166030884 disk.drbd1552.in0 disk.drbd1584.out0 network.recv164846 disk.nvme0n1.in 2.47265625 disk.drbd1356.in0 proc.cpu.threads29EE0+ _wandbruntime876  _runtime876W trainer/global_step1949 % _timestamp1.7857961640413387e+09 _runtime 876.864304808 _step152 & lr-AdamW/pg15.022999145986158e-05 & lr-AdamW/pg25.022999145986158e-05xܧ trainer/global_step1949 % _timestamp1.7857961640413387e+09 _runtime 876.864304808 _step152 & lr-AdamW/pg15.022999145986158e-05 & lr-AdamW/pg25.022999145986158e-0520+ _wandbruntime876  _runtime876H%O E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed35195159139840 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated20598383 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed7679545 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated20598383 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed20588727 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated35321897733632 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated34359230497188 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated5588922 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed5588667 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated35195263608832 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated32647897740832 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed32647880701472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current257 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated35195263608832 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 O@DeviceStats<Monitor.on_train_batch_start/requested_bytes.all.peak 2871874888 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated33479730545664 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated32647897740832 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed32647880701472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current257 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated35195263608832 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated33479730545664 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed35195159139840 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated20598383 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed7679545 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated20598383 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed20588727 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated35321897733632 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated34359230497188 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated5588922 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed5588667 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887j0+ _wandbruntime877  _runtime877c\&M QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed5591528 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed33576909211648 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated32664637913152 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed32664620873792 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 1712124426454 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated12965072 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated33496897308672 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed33496880269312 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed34376745300246 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed33496880269312 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 1716325253120 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed20599250 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated13275245 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated33573992 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated35340006096896 trainer/global_step1949 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1716412689920 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated33496897308672 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated34376848142842 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed35213205522432 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed12965070 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _step154 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed13275010 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated33573992 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1716412689920 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated35213309998592 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current233 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed33564320 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed20599250 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed35339844234752 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 1712210229690 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated20608920 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated12965072 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed35339844234752 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 1712210229690 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated20608920 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated12965072 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed32664620873792 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 1712124426454 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated12965072 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated33496897308672 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed33496880269312 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed34376745300246 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed33496880269312 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 1716325253120 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed20599250 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated13275245 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated33573992 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated35340006096896 trainer/global_step1949 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1716412689920 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated33496897308672 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated34376848142842 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed35213205522432 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed12965070 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _step154 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed13275010 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated33573992 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1716412689920 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated35213309998592 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current233 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed33564320 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed20599250 FDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated7876397 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated36227377497088 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed36227221919232 trainer/global_step1999 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 1759441768960 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 1807294658560 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated35240127576696 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated21126385 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated34338068696064 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated34416964 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed5732342 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed35240024734212 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated13290579 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed21116729 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed34338051656704 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated36097597894656 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed7876395 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed36097493425664 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1759529198592 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed34338051656704 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated21126385 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed13608737 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed34420057934848 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated34420082838528 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated13290579 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed21116729 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 1807163984384 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 1755135416740 _step157 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated33484906356832 % _timestamp1.7857961866764326e+09 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated7876397 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated36227377497088 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed36227221919232 trainer/global_step1999 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 1759441768960 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 1807294658560 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated35240127576696 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated21126385 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated34338068696064 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated34416964 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed5732342 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed35240024734212 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated13290579 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed21116729 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed34338051656704 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated36097597894656 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed7876395 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed36097493425664 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1759529198592 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed34338051656704 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated21126385 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed13608737 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed34420057934848 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated34420082838528 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated13290579 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed21116729 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 1807163984384 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 1755135416740 _step157 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated33484906356832 % _timestamp1.7857961866764326e+09 IDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1760408825344 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed36115539808256 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed5735149 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated35257745222350 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated33501646529152 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current245 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated7880334 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1760408825344 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed13615481 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed36245343847424 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated21136922 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 1808078795776 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 % _timestamp1.7857961870536456e+09 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed33501629489792 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated21136922 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed34355218419712 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1808213657088 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated36115644284416 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated13297222 _step158 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated34437289955328 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated34434144 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed13297220 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current "_ 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 1756098693198 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 1760321388544 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed21127252 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed34424472 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed34355218419712 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated34434144 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856=&M UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 1760321388544 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed21127252 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed34424472 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed34355218419712 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated34434144 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed21127252 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1760408825344 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed36115539808256 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed5735149 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated35257745222350 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated33501646529152 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current245 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated7880334 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1760408825344 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed13615481 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed36245343847424 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated21136922 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 1808078795776 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 % _timestamp1.7857961870536456e+09 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed33501629489792 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated21136922 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed34355218419712 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1808213657088 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated36115644284416 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated13297222 _step158 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated34437289955328 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated34434144 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed13297220 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 17560986931980+ _wandbruntime899  _runtime8995{ 0loss/FRt1/assignment_loss3.8261675997956157 0loss/FBt1/assignment_loss0.1437541163720501 -loss/FRt2/detection_loss3.34789057651704 1loss/SRqqt2/detection_loss1.9996664534670254 /loss/SRqqt2/assignment_loss1.9999869072535 0loss/FBt2/assignment_loss0.5344462157291774 1loss/SRqqt1/detection_loss1.8938030190551043 %loss/total_loss56.76281150920286 % _timestamp1.7857961870553453e+09 trainer/global_step1999 _step159 0loss/FRt2/assignment_loss4.7156243974512275 2loss/SRqqt1/assignment_loss1.4771725537125455 .loss/FRt1/detection_loss4.002797293336424 /loss/FBt1/detection_loss5.5442157012718924 /loss/FBt2/detection_loss27.277286675241257 epoch0 _runtime 899.878166409GB 0loss/FRt2/assignment_loss4.7156243974512275 2loss/SRqqt1/assignment_loss1.4771725537125455 .loss/FRt1/detection_loss4.002797293336424 /loss/FBt1/detection_loss5.5442157012718924 /loss/FBt2/detection_loss27.277286675241257 epoch0 _runtime 899.878166409 0loss/FRt1/assignment_loss3.8261675997956157 0loss/FBt1/assignment_loss0.1437541163720501 -loss/FRt2/detection_loss3.34789057651704 1loss/SRqqt2/detection_loss1.9996664534670254 /loss/SRqqt2/assignment_loss1.9999869072535 0loss/FBt2/assignment_loss0.5344462157291774 1loss/SRqqt1/detection_loss1.8938030190551043 %loss/total_loss56.76281150920286 % _timestamp1.7857961870553453e+09 trainer/global_step1999 _step159Ě(: ۬U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp37.0 gpu.0.powerWatts97.829' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent32.60966666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors09O(:  ۬U disk.drbd2118.in0 disk.sda.out 25.7109375 disk.sda1.out0 disk.drbd1226.out0 disk.drbd1965.in0 disk.sda2.in0 disk.rbd1.out 3.8828125 disk.drbd1289.out0) disk./.usagePercent32.34343387598743 disk.drbd2078.out0 disk.drbd2118.out0# disk.nvme0n1.out21.95068359375 disk.drbd1965.out0 disk.drbd2041.in0* proc.memory.percent17.182111740112305 disk.drbd1673.out0$ memory_percent41.22421741485596$ disk./.usageGB578.3334465026855 disk.sda1.in0 disk.drbd2041.out0 disk.sda.in0 disk.drbd1552.out0 disk.drbd1828.in0 disk.dm-0.out21.95068359375 disk.drbd1584.in0 disk.nvme0n1.in 2.47265625 disk.sda2.out 25.7109375 disk.drbd1817.out0 disk.drbd1550.in0 disk.drbd2078.in0! proc.memory.rssMB 11260.46875 network.recv169473 disk.drbd1550.out0 disk.drbd1226.in0 disk.drbd1817.in0 disk.drbd1931.out0 cpu14.27921094451311( proc.memory.availableMB 38519.296875 disk.rbd1.in0 disk.drbd1931.in0 disk.drbd1552.in0 disk.drbd1285.in0 disk.drbd1356.out0 disk.drbd2111.out0 disk.drbd1673.in0 disk.drbd1828.out0 disk.drbd1938.out0 disk.drbd1356.in0 disk.drbd1285.out0 disk.dm-0.in 2.47265625 disk.drbd1289.in0 disk.drbd1938.in0 disk.drbd2111.in0 disk.drbd1510.out0 disk.rbd0.in0 disk.rbd0.out0 proc.cpu.threads29 network.sent 1.925081e+06 disk.drbd1510.in0 disk.drbd1584.out03u](: ͥU gpu.0.gpu0.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts127.063' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.35433333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0'H(:  ݶU network.sent 1.959758e+06! disk.nvme0n1.out 23.017578125 disk.drbd1938.out0 disk.drbd2111.in0 disk.drbd1226.out0 disk.drbd1965.in0 disk.rbd0.out0 disk.drbd1285.in0 disk.dm-0.out 23.017578125 disk.drbd2078.in0 disk.drbd2118.out0 disk.drbd1356.in0 disk.drbd2111.out0 disk.drbd1673.in0 disk.drbd2041.in0 disk.drbd1938.in0) disk./.usagePercent32.34343067591981 disk.sda2.in0 disk.drbd1550.out0 disk.drbd1226.in0 proc.cpu.threads29 disk.nvme0n1.in 2.47265625 disk.drbd1356.out0 disk.drbd1552.in0 disk.rbd0.in0$ proc.memory.availableMB38519.75* proc.memory.percent17.182111740112305 disk.drbd1817.out0 disk.drbd2041.out0 disk.sda.out 26.1328125 disk.drbd1285.out0 disk.drbd1510.out0 disk.drbd1584.out0! proc.memory.rssMB 11260.46875 disk.sda1.out0 disk.drbd1673.out0 disk.dm-0.in 2.47265625 network.recv171559 disk.drbd1828.out0 disk.drbd1510.in0 disk.drbd1965.out0 disk.drbd1552.out0 disk.drbd1828.in0 disk.drbd1289.out0 disk.sda1.in0 disk.drbd2078.out0 disk.drbd1584.in0$ disk./.usageGB578.3333892822266 disk.sda2.out 26.1328125 disk.rbd1.out 3.8828125 disk.drbd1931.in0 disk.drbd1289.in0 disk.sda.in0 disk.drbd1817.in0 disk.drbd1550.in0 disk.drbd2118.in0$ memory_percent41.22352600097656 cpu14.274943780196148 disk.rbd1.in0 disk.drbd1931.out0Q0+ _wandbruntime921  _runtime9217$Q _step160 & lr-AdamW/pg15.280721010839219e-05 & lr-AdamW/pg25.280721010839219e-05 trainer/global_step2049 % _timestamp1.7857962088953009e+09 _runtime 921.718283167. & lr-AdamW/pg15.280721010839219e-05 & lr-AdamW/pg25.280721010839219e-05 trainer/global_step2049 % _timestamp1.7857962088953009e+09 _runtime 921.718283167 _step160$ٽ0+ _wandbruntime921  _runtime921c*'O @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 % _timestamp1.7857962088962498e+09 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated37132936674304 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed35280413774848 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed34321897933472 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated13622729 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed8073245 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated35196406846464 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed5875065 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated35196406846464 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed35196389807104 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed37132774804992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 1803525334016 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated21654387 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated35277116 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated13948570 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated5875323 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 _step161 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed35196389807104 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated35280438678528 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated21654387 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated13622729 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current258 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 1799023880248 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed36999827711488 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed13622727 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed36999827711488 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 1852361030144 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed21644731 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated34321914972832 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated36999932180480 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1803525334016 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed8073245 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated35196406846464 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 SGDeviceStratsMonitor.on_train_batch_start/inactive_split.small_pool.freed5875065 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated35196406846464 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed35196389807104 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed37132774804992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 1803525334016 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated21654387 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated35277116 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated13948570 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated5875323 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 _step161 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed35196389807104 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated35280438678528 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated21654387 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated13622729 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current258 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 1799023880248 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed36999827711488 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed13622727 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed36999827711488 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 1852361030144 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed21644731 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated34321914972832 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated36999932180480 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1803525334016 IDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated35213573609472 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed35297620891648 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated36138642301858 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated37017978570240 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated35213573609472 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed35284624 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 _step162 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated37017978570240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated35297645795328 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed36138539459262 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1804404960768 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed13955079 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed13629370 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated13955326 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed21655254 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed8077182 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed13629370 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1804404960768 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated8077184 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated35294296 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated13629372 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated35294296 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 trainer/global_step2049 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated5878142 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 1799987156706 _runtime 922.085929557 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated21664924 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed5877897 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed21655254 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0ۤ&M XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed35284624 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated13629372 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated37151043096064 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed37150889622528 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated5878142 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 1799987156706 _runtime 922.085929557 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated21664924 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed5877897 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed21655254 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated34338655145152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated35213573609472 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed35297620891648 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated36138642301858 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated37017978570240 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated35213573609472 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed35284624 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 _step162 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated37017978570240 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated35297645795328 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed36138539459262 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1804404960768 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed13955079 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed13629370 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated13955326 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed21655254 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed8077182 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed13629370 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1804404960768 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated8077184 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated35294296 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated13629372 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated35294296 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 trainer/global_step2049 EDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed37902161997312 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current236 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated35158923588832 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated8270097 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated13954879 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated13954879 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed13954877 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 1847434039808 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 1842998146880 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed13954877 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated38038461658112 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 1897532271104 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated36054744996864 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated14288861 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated37001921735712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated22182389 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed8270095 GADeviceStatsMonitor.on_train_batch_sta\rt/oversize_segments.current0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated37902266466304 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed36127610 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed14288623 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 1842912343756 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed35158906549472 _runtime 943.89865333K'O UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated36140794518528 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 1842912343756 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed35158906549472 _runtime 943.89865333 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed36054727957504 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed22172733 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 1847434039808 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed6018528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed22172733 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated36137268 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed37902161997312 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current236 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated35158923588832 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated8270097 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated13954879 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated13954879 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed13954877 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 1847434039808 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 1842998146880 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed13954877 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated38038461658112 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 1897532271104 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated36054744996864 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated14288861 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated37001921735712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated22182389 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed8270095 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated37902266466304 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed36127610 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed14288623 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 I6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 trainer/global_step2099 _step166 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed13961520 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed22183256 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1848401096192 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated22192926 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed22183256 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated22192926 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed14295367 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed37019436538770 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1848401096192 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated35175663761152 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated36154448 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed36144776 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated8274034 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed8274032 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated36154448 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current258 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current256 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated13961522 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated14295625 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated13961522 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed36071894720512L@M ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 trainer/global_step2099 _step166 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed13961520 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed22183256 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1848401096192 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated22192926 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed22183256 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated22192926 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed14295367 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed37019436538770 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1848401096192 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated35175663761152 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated36154448 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed36144776 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated8274034 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed8274032 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated36154448 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current258 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current256 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated13961522 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated14295625 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated13961522 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed36071894720512 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed36157976731648 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1898581984768 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 1843875620214 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 1848313659392 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed6021335 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated38056583620096 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed36144776 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 1898445026304 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated37019539381366 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2"'0+ _wandbruntime944  _runtime944V .loss/FRt1/detection_loss3.303256017493678 -loss/FBt1/detection_loss4.91726635012972 0loss/FRt1/assignment_loss3.9124659234350854 /loss/FRt2/assignment_loss4.605963818014485 2loss/SRqqt1/assignment_loss1.7446284782959194 trainer/global_step2099 % _timestamp1.7857962314487264e+09 _step167 0loss/FBt2/assignment_loss0.4333682379219681 1loss/SRqqt2/detection_loss2.3258584839376018 .loss/FBt2/detection_loss10.52114704743217 %loss/total_loss39.58142015930724 /loss/FRt2/detection_loss3.9534230658861054 1loss/SRqqt1/detection_loss1.8047523370539327 _runtime 944.271558269 epoch0 1loss/SRqqt2/assignment_loss1.902155997024642 0loss/FBt1/assignment_loss0.1571344026819325v 0loss/FBt2/assignment_loss0.4333682379219681 1loss/SRqqt2/detection_loss2.3258584839376018 .loss/FBt2/detection_loss10.52114704743217 %loss/total_loss39.58142015930724 /loss/FRt2/detection_loss3.9534230658861054 1loss/SRqqt1/detection_loss1.8047523370539327 _runtime 944.271558269 epoch0 1loss/SRqqt2/assignment_loss1.902155997024642 0loss/FBt1/assignment_loss0.1571344026819325 .loss/FRt1/detection_loss3.303256017493678 -loss/FBt1/detection_loss4.91726635012972 0loss/FRt1/assignment_loss3.9124659234350854 /loss/FRt2/assignment_loss4.605963818014485 2loss/SRqqt1/assignment_loss1.7446284782959194 trainer/global_step2099 % _timestamp1.7857962314487264e+09 _step167閐(: U gpu.0.gpu21.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts106.152' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent35.384 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0gvYS(:  V disk.drbd1285.in0 disk.dm-0.in 2.47265625 disk.drbd1552.out0 disk.drbd1817.out0 disk.drbd1931.out0 disk.rbd1.in0) proc.memory.percent17.18212366104126 disk.drbd1584.in0 disk.sda2.in0 disk.drbd2111.out0* proc.memory.availableMB38519.42578125$ disk./.usageGB578.3334541320801 disk.drbd1510.out0 disk.drbd1356.in0 proc.cpu.threads29 disk.drbd1673.in0 disk.drbd1673.out0 disk.drbd1285.out0 disk.drbd1931.in0 disk.drbd2041.in0 disk.drbd1938.in0 disk.sda1.out0 disk.drbd1510.in0 disk.drbd1817.in0 disk.drbd1550.out0# memory_percent41.2240207195282) disk./.usagePercent32.34343430266311 disk.sda.in0" disk.nvme0n1.out 23.4560546875 disk.sda2.out 26.8984375 disk.drbd1828.in0 disk.drbd1289.out0 disk.drbd1938.out0 disk.drbd1584.out0 disk.drbd2118.out0 disk.drbd1965.out0 disk.drbd1828.out0 network.recv176345 disk.rbd0.in0 disk.nvme0n1.in 2.47265625 disk.drbd1226.in0 disk.drbd1226.out0 disk.drbd2078.out0# proc.memory.rssMB 11260.4765625 network.sent 2.049703e+06 disk.drbd2111.in0 disk.drbd1552.in0 disk.drbd2078.in0 disk.drbd2118.in0 disk.drbd1356.out0 disk.drbd2041.out0 disk.drbd1550.in0 disk.drbd1289.in0 cpu14.269738407589518 disk.sda1.in0 disk.rbd0.out0 disk.sda.out 26.8984375 disk.dm-0.out 23.4560546875 disk.rbd1.out 3.9150390625 disk.drbd1965.in0PD(: U gpu.0.gpu28.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts121.418' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent40.47266666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0HGQ(:  źV disk.dm-0.out 23.7490234375 disk.drbd1510.in0) proc.memory.percent17.18212366104126 cpu14.26556236012987 disk.drbd1356.out0 disk.drbd2111.out0# proc.memory.rssMB 11260.4765625' proc.memory.availableMB 38520.09375 network.recv177421 disk.drbd1673.in0 disk.drbd2078.in0$ disk./.usageGB578.3334350585938 disk.drbd1552.out0 disk.drbd1285.out0 disk.drbd1584.out0 disk.drbd1584.in0 disk.drbd1817.out0 disk.drbd1552.in0 disk.drbd1550.in0 network.sent 2.052707e+06 disk.sda.out 27.3984375 disk.drbd2078.out0 disk.drbd2118.out0$ memory_percent41.22300148010254* disk./.usagePercent32.343433235973905 disk.drbd1550.out0 proc.cpu.threads29 disk.rbd1.in0 disk.drbd1673.out0 disk.drbd1931.in0 disk.drbd1931.out0 disk.drbd1289.in0 disk.drbd1938.in0 disk.sda2.in0 disk.rbd1.out 4.0087890625 disk.rbd0.out0 disk.drbd2041.in0 disk.sda1.out0 disk.drbd1226.in0 disk.drbd1226.out0 disk.nvme0n1.in 2.47265625 disk.rbd0.in0 disk.drbd1965.out0" disk.nvme0n1.out 23.7490234375 disk.drbd2111.in0 disk.sda2.out 27.3984375 disk.drbd1817.in0 disk.drbd1828.in0 disk.drbd1289.out0 disk.sda1.in0 disk.dm-0.in 2.47265625 disk.drbd1828.out0 disk.drbd1510.out0 disk.drbd1965.in0 disk.drbd2118.in0 disk.drbd1356.in0 disk.drbd1938.out0 disk.drbd2041.out0 disk.sda.in0 disk.drbd1285.in0l"0+ _wandbruntime966  _runtime966:lc _step168 ' lr-AdamW/pg15.5384428756922805e-05 ' lr-AdamW/pg25.5384428756922805e-05 trainer/global_step2149 $ _timestamp1.785796253758051e+09 _runtime 966.581006379x+iH ' lr-AdamW/pg15.5384428756922805e-05 ' lr-AdamW/pg25.5384428756922805e-05 trainer/global_step2149 $ _timestamp1.785796253758051e+09 _runtime 966.581006379 _step16850+ _wandbruntime966  _runtime966'΃'O \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed37882715972736 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed35995915165472 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed38804496283136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated6162661 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed36913066107904 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed37001125454848 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 1942848501760 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1891517604864 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated8466947 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 1886886610388 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated36913083147264 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed36913066107904 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated37882818815220 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 1891430175232 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated36997420 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated38804600752128 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed38804496283136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 1886800807264 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed14629356 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated37001150358528 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed36987762 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 trainer/global_step2149 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed8466945 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed14287027 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated22710391 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed22700735 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 145092096 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 1942728313344 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed38943853768192}'O PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated22710391 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed22700735 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 145092096 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 1942728313344 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed38943853768192 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed37882715972736 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed35995915165472 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed38804496283136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated6162661 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed36913066107904 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed37001125454848 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 1942848501760 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1891517604864 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated8466947 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 1886886610388 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated36913083147264 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed36913066107904 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated37882818815220 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 1891430175232 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated36997420 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated38804600752128 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed38804496283136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 1886800807264 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed14629356 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated37001150358528 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed36987762 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 trainer/global_step2149 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed8466945 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed14287027 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 0+ _wandbruntime966  _runtime966/xM [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 _step170 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated8470884 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1943761929728 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated37014600 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated14636392 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.[Usmall_pool.peak 250523136 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed22711258 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated37014600 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated36930249910272 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed37004928 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated14293672 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated38822647141888 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed14636134 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1892397231616 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated36012672377152 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed36012655337792 trainer/global_step2149 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 1887678280486 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated36930249910272 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 1892309794816 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current258 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated6165508 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated14293672 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed6165252&M KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated38822647141888 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed14636134 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1892397231616 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated36012672377152 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed36012655337792 trainer/global_step2149 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 1887678280486 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated36930249910272 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 1892309794816 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current258 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated6165508 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated14293672 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed6165252 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 _step170 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated8470884 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 1943761929728 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated37014600 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated14636392 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed22711258 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated37014600 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated36930249910272 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed37004928 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated14293672 EDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated14969935 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed39849396958208 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed38763613052244 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 1930689270772 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed37847914 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 _runtime 988.332706465 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed39706830568960 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed23228737 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current241 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed39706830568960 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed14969692 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 trainer/global_step2199 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated37771421297664 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated6306138 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated23238393 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated37861506198528 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 1930775073896 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 1935513740288 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated23238393 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed14619177 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated36832940820832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed37771404258304 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated39849556730368 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed36832923781472 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 _step173 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated38763715894728 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated39706935037952 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed23228737 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated37771421297664 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 1935426310656 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed8663795 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 1988050531840 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated14619179 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated14969935 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed39849396958208 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed38763613052244 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 1930689270772 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed37847914 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 _runtime 988.332706465 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed39706830568960 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed23228737 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current241 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed39706830568960 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed14969692 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 trainer/global_step2199 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated37771421297664 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated6306138 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated23238393 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated37861506198528 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 1930775073896 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 1935513740288 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated23238393 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed14619177 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated36832940820832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed37771404258304 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated39849556730368 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed36832923781472 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 _step173 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated38763715894728 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated39706935037952 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed23228737 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated37771421297664 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360?)0+ _wandbruntime988  _runtime988>&M E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed38781230697786 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated39867668739072 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1936393367040 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed8667732 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated36849680993152 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 ODeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated37788588060672 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed37788571021312 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated37874752 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated14625822 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated8667734 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 1936305930240 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated37878713315328 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated23248930 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current246 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 168153600 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated38781333540382 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1936393367040 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 trainer/global_step2199 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated37874752 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated37788588060672 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated39724981427712 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 % _timestamp1.7857962758819914e+09 _step174 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed37878688411648 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed37788571021312 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed39724876951552 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 1936305930240 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 1931652547230 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed14976450 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 1988812173824 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed23239260 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated6308962 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed14625820 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated23248930 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 143249920 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed37865080 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed6308718 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current244 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed36849663953792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251ZT&M QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 1936305930240 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated37878713315328 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated23248930 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current246 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 168153600 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated38781333540382 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1936393367040 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 trainer/global_step2199 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated37874752 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated37788588060672 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated39724981427712 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 % _timestamp1.7857962758819914e+09 _step174 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed37878688411648 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed37788571021312 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed39724876951552 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 1936305930240 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 1931652547230 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed14976450 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 1988812173824 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed23239260 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated6308962 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed14625820 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated23248930 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 143249920 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed37865080 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed6308718 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current244 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed36849663953792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed38781230697786 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated39867668739072 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 1936393367040 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed8667732 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated36849680993152 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 ODeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated37788588060672 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed37788571021312 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated37874752 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated14625822 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated8667734 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0H!0+ _wandbruntime988  _runtime988m 1loss/FBt1/assignment_loss0.17852172116720888 0loss/FBt2/assignment_loss0.3144430423746079 .loss/FRt2/detection_loss3.765788305600484 1loss/SRqqt1/detection_loss1.8841642139821617 0loss/SRqqt2/detection_loss2.195943018200596 .loss/FRt2/assignment_loss4.59251792271932 -loss/FBt1/detection_loss6.74934254142262 .loss/FBt2/detection_loss8.315605902100673 $loss/total_loss38.2898460216949 epoch0 trainer/global_step2199 % _timestamp1.7857962758839312e+09 _step175 /loss/FRt1/detection_loss2.8861210106067285 _runtime 988.706775958 0loss/FRt1/assignment_loss3.8316294313634485 2loss/SRqqt1/assignment_loss1.7059949921890043 1loss/SRqqt2/assignment_loss1.869773919968044  _runtime 988.706775958 0loss/FRt1/assignment_loss3.8316294313634485 2loss/SRqqt1/assignment_loss1.7059949921890043 1loss/SRqqt2/assignment_loss1.869773919968044 1loss/FBt1/assignment_loss0.17852172116720888 0loss/FBt2/assignment_loss0.3144430423746079 .loss/FRt2/detection_loss3.765788305600484 1loss/SRqqt1/detection_loss1.8841642139821617 0loss/SRqqt2/detection_loss2.195943018200596 .loss/FRt2/assignment_loss4.59251792271932 -loss/FBt1/detection_loss6.74934254142262 .loss/FBt2/detection_loss8.315605902100673 $loss/total_loss38.2898460216949 epoch0 trainer/global_step2199 % _timestamp1.7857962758839312e+09 _step175 /loss/FRt1/detection_loss2.8861210106067285(: ۵U gpu.0.gpu38.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts125.791' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent41.93033333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0붌'(:  U network.sent 2.142654e+06 disk.drbd2078.out0 disk.drbd1356.out0 disk.sda2.in0 disk.sda2.out 28.01171875 disk.drbd1550.in0 disk.drbd1938.in0 disk.drbd1356.in0 disk.drbd2041.in0 disk.rbd0.in0 disk.drbd1817.in0 disk.drbd1550.out0 disk.sda.out 28.01171875 disk.drbd1931.in0 disk.drbd1828.out0 disk.drbd1817.out0 disk.drbd2041.out0 disk.drbd1938.out0 disk.drbd1510.out0 cpu14.245717597956338 disk.drbd1673.in0$ disk./.usageGB578.3334503173828 disk.drbd1289.out0) proc.memory.percent17.18212366104126 disk.drbd1285.in0 disk.drbd1226.in0 disk.drbd2118.in0 disk.drbd1965.in0 disk.drbd1931.out0 disk.drbd1510.in0# proc.memory.rssMB 11260.4765625 disk.drbd1584.out0 disk.rbd1.out 4.0712890625 disk.drbd1828.in0 disk.drbd1289.in0% memory_percent41.223472356796265 proc.cpu.threads29 disk.nvme0n1.in 2.47265625 disk.drbd2111.in0 disk.sda1.in0 disk.drbd1552.in0 disk.sda1.out0 disk.drbd1226.out0 disk.drbd2078.in0 disk.rbd1.in0 network.recv182288 disk.rbd0.out0 disk.drbd2118.out0# disk.nvme0n1.out24.14794921875 disk.drbd2111.out0) disk./.usagePercent32.34343408932527 disk.drbd1584.in0 disk.drbd1673.out0 disk.drbd1552.out0 /disk.drbd1285.out0 disk.dm-0.in 2.47265625 disk.dm-0.out24.14794921875* proc.memory.availableMB38519.78515625 disk.drbd1965.out0 disk.sda.in0"z(: ĭU gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts94.367' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent31.45566666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0eW(:  ĭV disk.drbd1289.out0 disk.rbd1.in0 disk.drbd2041.in0 disk.drbd1356.in0 disk.sda2.in0 disk.drbd1931.in0 disk.drbd1226.in0 cpu14.236792932027694 proc.cpu.threads29# disk.nvme0n1.out24.43017578125 disk.drbd1552.in0 disk.drbd1289.in0 disk.drbd1285.in0 disk.drbd1226.out0 disk.drbd2111.out0$ memory_percent41.22318625450134 disk.drbd1965.in0 disk.drbd1673.out0 disk.drbd1550.in0) proc.memory.percent17.18212366104126 disk.drbd2078.in0 disk.drbd2111.in0 network.recv182288 disk.drbd2041.out0 disk.drbd1510.in0 disk.drbd1552.out0 disk.rbd0.out0 disk.drbd2078.out0 disk.drbd1931.out0 disk.drbd1550.out0 disk.drbd1828.in0 disk.drbd1285.out0 disk.drbd1817.out0 disk.drbd1938.out0 disk.sda.in0 disk.drbd1356.out0 network.sent 2.142654e+06 disk.drbd1817.in0 disk.drbd1584.out0* proc.memory.availableMB38519.97265625) disk./.usagePercent32.34343515601448$ disk./.usageGB578.3334693908691 disk.rbd1.out 4.103515625 disk.dm-0.out24.43017578125 disk.drbd2118.in0# proc.memory.rssMB 11260.4765625 disk.dm-0.in 2.47265625 disk.drbd1584.in0 disk.drbd2118.out0 disk.drbd1938.in0 disk.rbd0.in0 disk.drbd1965.out0 disk.drbd1673.in0 disk.sda1.in0 disk.drbd1828.out0 disk.nvme0n1.in 2.47265625 disk.sda2.out 28.31640625 disk.sda.out 28.31640625 disk.sda1.out0 disk.drbd1510.out0y2- _wandbruntime1010  _runtime1010ö1 % _timestamp1.7857962977380795e+09 _runtime 1010.56104146 _step176 ' lr-AdamW/pg15.7961647405453414e-05 ' lr-AdamW/pg25.7961647405453414e-05 trainer/global_step2249@\ ' lr-AdamW/pg15.7961647405453414e-05 ' lr-AdamW/pg25.7961647405453414e-05 trainer/global_step2249 % _timestamp1.7857962977380795e+09 _runtime 1010.56104146 _step176O2- _wandbruntime1010  _runtime10100'O =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 2033069776896 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated38629759448064 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated15310648 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated8860647 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed39644510131752 % _timestamp1.7857962977390306e+09 _step177 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed38721837134848 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated6450001 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated37669949436832 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2033200451072 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current239 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated38629759448064 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed40754906911744 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated39644612974236 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed23756739 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 trainer/global_step2249 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed38629742408704 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated40755062489600 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed8860645 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1979509875712 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed40609164854784 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated14951329 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed23756739 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed40609164854784 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed6449764 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed15310409 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current237 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 1979422446080 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed14951327 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 1979422446080 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed37669932397472 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed14951327 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated23766395 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 T5'O @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed39644510131752 % _timestamp1.7857962977390306e+09 _step177 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed38721837134848 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated6450001 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated37669949436832 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2033200451072 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current239 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated38629759448064 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed40754906911744 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated39644612974236 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed23756739 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 trainer/global_step2249 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed38629742408704 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated40755062489600 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed8860645 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 1979509875712 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed40609164854784 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated14951329 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed23756739 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed40609164854784 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed6449764 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed15310409 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current237 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 1979422446080 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed14951327 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 1979422446080 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed37669932397472 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed14951327 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated23766395 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 2033069776896 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated38629759448064 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated15310648 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated8860647!2- _wandbruntime1010  _runtime1010xYoq%M HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed39662127777294 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 trainer/global_step2249 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated14957972 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated38646926211072 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated38734904 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed15317183 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated38646926211072 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed38725232 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated23776932 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated39662230619890 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated6452875 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated40773174965248 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated15317459 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated38734904 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current274 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 % _timestamp1.7857962981220605e+09 _step178 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed14957970 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated23776932 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated14957972 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed38646909171712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed23767262 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated40627315713536 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1980389502464 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current276 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 2034105809920 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed37686672569792 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 _runtime1010.945398135 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed23767262 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 2033968851456 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated37686689609152 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_poo\_{Sl.allocated121 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 1980302065664 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed38739044251648t&M RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 % _timestamp1.7857962981220605e+09 _step178 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed14957970 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated23776932 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated14957972 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed38646909171712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed23767262 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated40627315713536 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 1980389502464 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current276 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 2034105809920 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed37686672569792 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 _runtime1010.945398135 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed23767262 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 2033968851456 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated37686689609152 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 1980302065664 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed38739044251648 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed39662127777294 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 trainer/global_step2249 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated14957972 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated38646926211072 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated38734904 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed15317183 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated38646926211072 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed38725232 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated23776932 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated39662230619890 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated6452875 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated40773174965248 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated15317459 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated38734904 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current274 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 27682406406լ2- _wandbruntime1010  _runtime1010 % _timestamp1.7857962981237369e+09 _step179 .loss/FRt1/assignment_loss3.90236704826355 /loss/FRt2/assignment_loss4.681993007659912 2loss/SRqqt2/assignment_loss1.8510107896595358 ,loss/FRt2/detection_loss4.1113525852561 .loss/FRt1/detection_loss2.951520178318024 1loss/SRqqt1/detection_loss2.2552730893531847 _runtime1010.946570737 2loss/SRqqt1/assignment_loss1.5650035472710928 %loss/total_loss40.72970390196843 epoch0 trainer/global_step2249 1loss/FBt1/assignment_loss0.26335325972468127 1loss/FBt2/assignment_loss0.15427778648154344 1loss/SRqqt2/detection_loss2.4534442720591656 .loss/FBt1/detection_loss5.321845357539991 /loss/FBt2/detection_loss11.2182629803816476x % _timestamp1.7857962981237369e+09 _step179 .loss/FRt1/assignment_loss3.90236704826355 /loss/FRt2/assignment_loss4.681993007659912 2loss/SRqqt2/assignment_loss1.8510107896595358 ,loss/FRt2/detection_loss4.1113525852561 .loss/FRt1/detection_loss2.951520178318024 1loss/SRqqt1/detection_loss2.2552730893531847 _runtime1010.946570737 2loss/SRqqt1/assignment_loss1.5650035472710928 %loss/total_loss40.72970390196843 epoch0 trainer/global_step2249 1loss/FBt1/assignment_loss0.26335325972468127 1loss/FBt2/assignment_loss0.15427778648154344 1loss/SRqqt2/detection_loss2.4534442720591656 .loss/FBt1/detection_loss5.321845357539991 /loss/FBt2/detection_loss11.218262980381647 [v(: ӭɮU gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts117.623' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent39.20766666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0i3U(:  ӭU disk.drbd2118.in0 network.sent 2.190599e+06$ disk./.usageGB578.3334503173828 disk.drbd1289.out0$ memory_percent41.22316241264343 disk.drbd1965.in0 disk.rbd0.out0 disk.drbd2111.out0) disk./.usagePercent32.34343408932527 disk.drbd1938.in0 disk.rbd0.in0 disk.drbd1285.in0 disk.dm-0.out24.67236328125 disk.drbd1817.in0 disk.drbd1938.out0 disk.sda.out 28.7578125# disk.nvme0n1.out24.67236328125 disk.drbd2111.in0* proc.memory.availableMB38519.98828125 disk.rbd1.in0 disk.drbd1552.out0 disk.drbd1510.out0 disk.drbd1584.out0 disk.drbd2078.in0 disk.drbd2118.out0 disk.drbd1931.in0 disk.drbd1510.in0 disk.sda2.in0 disk.drbd1817.out0 disk.drbd1673.out0 disk.drbd1552.in0 disk.drbd1550.out0 disk.sda1.out0 disk.sda2.out 28.7578125) proc.memory.percent17.18212366104126 disk.drbd2041.in0 disk.dm-0.in 2.47265625 disk.drbd2078.out0 disk.nvme0n1.in 2.47265625 proc.cpu.threads29 network.recv186077 disk.rbd1.out 4.283203125 disk.drbd1965.out0 disk.drbd1828.in0# proc.memory.rssMB 11260.4765625 cpu14.227429686110453 disk.drbd1931.out0 disk.drbd2041.out0 disk.drbd1289.in0 disk.sda.in0 disk.sda1.in0 disk.drbd1584.in0 disk.drbd1226.out0 disk.drbd1356.in0 disk.drbd1356.out0 disk.drbd1550.in0 disk.drbd1226.in0 disk.drbd1285.out0 disk.drbd1673.in0 disk.drbd1828.out0852- _wandbruntime1032  _runtime1032_" _step180 ' lr-AdamW/pg15.9250256729718725e-05 ' lr-AdamW/pg25.9250256729718725e-05 trainer/global_step2299 % _timestamp1.7857963196855605e+09 _runtime1032.510713981? ' lr-AdamW/pg15.9250256729718725e-05 ' lr-AdamW/pg25.9250256729718725e-05 trainer/global_step2299 % _timestamp1.7857963196855605e+09 _runtime1032.510713981 _step180դ2- _wandbruntime1032  _runtime1032tp'O QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed39488080559104 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 2078279569920 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed24284741 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed9057495 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 2023418581504 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 _step181 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed39488080559104 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed24284741 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current224 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 _runtime1032.510800977 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated39577876 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 trainer/global_step2299 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed15651795 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated39488097598464 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed15283477 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2023506011136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated6594522 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed39568218 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2023418581504 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated24294397 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2023506011136 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current222 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed6594300 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed39582192974848 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated38506958052832 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated39582217878528 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated15283479 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed41511499140608 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed15283477 % _timestamp1.7857963196862311e+09 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed41511499140608 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2078414438400 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed41660472544768 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated38506958052832 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated39582217878528 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated15283479 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed41511499140608 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed15283477 % _timestamp1.7857963196862311e+09 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed41511499140608 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2078414438400 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed41660472544768 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 2023418581504 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 _step181 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed39488080559104 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed24284741 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current224 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 _runtime1032.510800977 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated39577876 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 trainer/global_step2299=2- _wandbruntime1032  _runtime1032_b%&M WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2024298201088 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 126472704 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated39505264361472 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed39505247322112 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 2024298201088 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed41529545523200 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed39599400091648 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed39585384 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated39595056 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 2079329611264 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 trainer/global_step2299 _step182 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated6597361 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 151376384 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed40543024856802 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _runtime1032.883084333 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated9061434 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed15658547 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed39585384 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated24304934 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 % _timestamp1.7857963200596905e+09 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed6597115 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated24304934 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated39505264361472 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated40543127699398 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated38523698225152 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed15290120 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2024385637888 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current248 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 2079203138560 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated15290122 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed15290120 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated39595056 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0&M SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 126472704 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated39505264361472 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed39505247322112 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 2024298201088 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed41529545523200 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed39599400091648 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed39585384 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated39595056 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 2079329611264 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 trainer/global_step2299 _step182 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated6597361 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 151376384 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed40543024856802 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _runtime1032.883084333 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated9061434 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed15658547 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed39585384 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated24304934 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 % _timestamp1.7857963200596905e+09 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed6597115 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated24304934 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated39505264361472 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated40543127699398 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated38523698225152 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed15290120 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2024385637888 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current248 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 2079203138560 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated15290122 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed15290120 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated39595056 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2024298201088n;?2- _wandbruntime1032  _runtime1032lC 1loss/FBt2/assignment_loss0.20204861451873932 .loss/FRt2/detection_loss4.110324717502095 1loss/SRqqt1/detection_loss1.2262542730939838 %loss/total_loss39.89484802288648 epoch0 /loss/FRt1/assignment_loss4.122931957244873 /loss/FRt1/detection_loss3.3653836172251475 1loss/SRqqt2/detection_loss2.6890461710675715 /loss/FBt2/detection_loss10.230916729340187 trainer/global_step2299 _step183 2loss/SRqqt1/assignment_loss1.8245084954555644 2loss/SRqqt2/assignment_loss1.9151598525808213 .loss/FRt2/assignment_loss4.54745833553485 1loss/FBt1/assignment_loss0.19213812324742321 .loss/FBt1/detection_loss5.468677136075227 $ _timestamp1.785796320061585e+09 _runtime1032.884532601fǶ .loss/FRt2/assignment_loss4.54745833553485 1loss/FBt1/assignment_loss0.19213812324742321 .loss/FBt1/detection_loss5.468677136075227 $ _timestamp1.785796320061585e+09 _runtime1032.884532601 1loss/FBt2/assignment_loss0.20204861451873932 .loss/FRt2/detection_loss4.110324717502095 1loss/SRqqt1/detection_loss1.2262542730939838 %loss/total_loss39.89484802288648 epoch0 /loss/FRt1/assignment_loss4.122931957244873 /loss/FRt1/detection_loss3.3653836172251475 1loss/SRqqt2/detection_loss2.6890461710675715 /loss/FBt2/detection_loss10.230916729340187 trainer/global_step2299 _step183 2loss/SRqqt1/assignment_loss1.8245084954555644 2loss/SRqqt2/assignment_loss1.9151598525808213](: ޅU gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts107.673' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent35.891 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0IXX(:  U proc.cpu.threads29 disk.sda2.out 28.98046875 disk.drbd1673.in0 disk.rbd0.in0 disk.drbd1584.in0 disk.drbd1550.out0 disk.drbd1938.out0 disk.drbd1584.out0* disk./.usagePercent32.343430889257654 disk.drbd2111.in0$ memory_percent41.22379422187805# disk.nvme0n1.out24.85693359375 disk.drbd1965.in0 disk.drbd2041.out0 disk.drbd1550.in0# proc.memory.rssMB 11260.4765625 disk.drbd1356.in0 disk.rbd1.in0 disk.drbd1931.out0 disk.drbd1828.out0 disk.drbd1289.in0 disk.drbd2118.out0 disk.drbd1817.out0 disk.drbd2041.in0 disk.sda2.in0 disk.drbd1965.out0 disk.drbd1552.out0 disk.drbd1510.in0 disk.drbd2111.out0 disk.drbd1552.in0 disk.drbd1289.out0 disk.drbd1285.out0 disk.drbd2118.in0 disk.sda.in0 disk.sda1.in0 disk.sda1.out0 disk.drbd1817.in0 disk.drbd2078.out0 disk.nvme0n1.in 2.47265625 disk.drbd1931.in0 disk.drbd1828.in0 disk.drbd1226.out0 disk.rbd1.out 4.345703125 disk.dm-0.out24.85693359375 disk.drbd1226.in0 cpu14.218773695897063 network.sent 2.190599e+06 disk.drbd1673.out0 disk.sda.out 28.98046875 disk.drbd2078.in0 network.recv186077 disk.rbd0.out0 disk.drbd1285.in0* proc.memory.availableMB38519.57421875$ disk./.usageGB578.3333930969238 disk.dm-0.in 2.47265625) proc.memory.percent17.18212366104126 disk.drbd1356.out0 disk.drbd1938.in0 disk.drbd1510.out0)֙(: U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts99.26' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent33.08666666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0w?V(:  ɁV) disk./.usagePercent32.34343408932527 disk.drbd1931.out0 proc.cpu.threads29 network.recv188567 disk.drbd1828.in0 disk.sda2.in0) proc.memory.percent17.18212366104126 disk.drbd2041.out0 disk.drbd1550.out0 disk.sda.in0 disk.sda1.in0 disk.drbd1552.out0 disk.dm-0.out25.05224609375 disk.drbd1226.out0 disk.drbd2118.in0 disk.drbd1931.in0 disk.drbd1828.out0 disk.rbd0.in0 disk.sda.out 29.29296875 disk.drbd1817.in0 disk.drbd1965.out0 disk.drbd1289.in0 disk.drbd1289.out0 disk.drbd2118.out0 disk.drbd1356.in0 disk.drbd2111.in0$ disk./.usageGB578.3334503173828 disk.drbd1550.in0 disk.drbd1584.out0 disk.drbd2111.out0# proc.memory.rssMB 11260.4765625 cpu14.209111447290294 disk.drbd1285.in0 disk.drbd1510.in0 disk.dm-0.in 2.47265625 disk.drbd1226.in0 disk.drbd1356.out0 network.sent 2.238304e+06 disk.drbd1965.in0 disk.drbd1817.out0% memory_percent41.223931312561035 disk.drbd1673.in0 disk.drbd1673.out0 disk.drbd1552.in0 disk.drbd1938.out0 disk.rbd0.out0 disk.sda2.out 29.29296875( proc.memory.availableMB 38519.484375 disk.drbd2041.in0 disk.drbd2078.in0 disk.drbd1938.in0 disk.sda1.out0 disk.drbd2078.out0 disk.drbd1285.out0 disk.drbd1584.in0# disk.nvme0n1.out25.05224609375 disk.drbd1510.out0 disk.nvme0n1.in 2.47265625 disk.rbd1.in0 disk.rbd1.out 4.3457031252- _wandbruntime1054  _runtime10549u  % _timestamp1.7857963416888213e+09 _runtime1054.511780748 _step184 & lr-AdamW/pg16.053886605398402e-05 & lr-AdamW/pg26.053886605398402e-05 trainer/global_step2349ڌ _runtime1054.511780748 _step184 & lr-AdamW/pg16.053886605398402e-05 & lr-AdamW/pg26.053886605398402e-05 trainer/global_step2349 % _timestamp1.7857963416888213e+09[<2- _wandbruntime1054  _runtime1054pO L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 2067414716928 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 _step185 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed15615627 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated40346435748864 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2067502146560 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed24812743 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated40442573718528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated24822399 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current239 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 trainer/global_step2349 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated15615629 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed15615627 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed15990501 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed42413833426432 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed6736156 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed42566129492992 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 2062440464420 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2067502146560 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2067414716928 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed41406304290768 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed40442548814848 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated15990740 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552% A<'O PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated24822399 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current239 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 trainer/global_step2349 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated15615629 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed15615627 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed15990501 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed42413833426432 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed6736156 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed42566129492992 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 2062440464420 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2067502146560 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2067414716928 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed41406304290768 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed40442548814848 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated15990740 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 2067414716928 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 _step185 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed15615627 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated40346435748864 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2067502146560 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed24812743 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated40442573718528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0/䥵2- _wandbruntime1054  _runtime1054CpX&M O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated40455208 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated40455208 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated15997542 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2068381773312 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed41423921936310 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 % _timestamp1.7857963420507398e+09 _runtime1054.874086828 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated42584401978368 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed15622270 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated24832936 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 2124621143040 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated41424024778906 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed15997287 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated39360706841152 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2068294336512 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 2124488378880 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed24823266 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated42431984285184 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated40455208 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated40455208 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated15997542 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2068381773312 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed41423921936310 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 % _timestamp1.7857963420507398e+09 _runtime1054.874086828 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated42584401978368 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed15622270 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated24832936 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 2124621143040 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated41424024778906 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed15997287 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated39360706841152 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2068294336512 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 2124488378880 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed24823266 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated42431984285184 EDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 trainer/global_step2399 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2111498281984 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed43471688507904 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated25350401 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed15947777 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 2168783853056 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated6880749 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 2111410852352 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed9451195 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed43316167712256 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated25350401 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated16331946 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed42287201370276 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated43471844085760 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 _runtime1076.628569979 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed41288522 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated43316272181248 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current257 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed41302904654848 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed40180958245472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed6880494 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated9451197 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current255 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated15947779 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed15947777 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2111498281984 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2111410852352 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed16331689 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024吃'O UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated41204773899264 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated41302929558528 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2111410852352 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed16331689 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated42287304212760 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed41204756859904 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2168914527232 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated41204773899264 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed25340745 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed25340745 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 # _timestamp1.78579636380537e+09 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 2106243124804 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated43316272181248 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed41204756859904 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 trainer/global_step2399 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2111498281984 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed43471688507904 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated25350401 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed15947777 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 2168783853056 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated6880749 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 2111410852352 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed9451195 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed43316167712256 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated25350401 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated16331946 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed42287201370276 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated43471844085760 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 _runtime1076.628569979 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed41288522 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated43316272181248 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current257 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed41302904654848 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed40180958245472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed6880494 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated9451197 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current255 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated15947779 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed15947777 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2111498281984 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 BM OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated16338753 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated43334318571008 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated41221940662272 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed25351268 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed16338502 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current249 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed42304819015818 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated15954422 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed41320111771648 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 2169682788352 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated25360938 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated6883619 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UGDevimhceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _step190 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated41315360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed25351268 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed40197698417792 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed41221923622912 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated9455134 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated43489952227840 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed15954420 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed41221923622912 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated15954422 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated43334318571008 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated41320136675328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed15954420 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed41305688 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated42304921858414 _runtime 1076.98919465 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 2112290471936 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed41305688 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2112377908736 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2112290471936 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated41315360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed43334214094848 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 2112377908736 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 2107120598026 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed6883370 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2107206401262 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _step190 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated41315360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed25351268 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed40197698417792 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed41221923622912 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated9455134 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated43489952227840 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed15954420 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed41221923622912 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated15954422 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated43334318571008 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated41320136675328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed15954420 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed41305688 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated42304921858414 _runtime 1076.98919465 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 2112290471936 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed41305688 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2112377908736 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2112290471936 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated41315360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed43334214094848 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 2112377908736 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 2107120598026 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed6883370 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2107206401262 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated41221940662272 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed25351268 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed16338502 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current249 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed42304819015818 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated15954422 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed41320111771648 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 2169682788352 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated25360938 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated6883619M3L/2- _wandbruntime1076  _runtime1076'FbM .loss/FRt1/assignment_loss4.05896008840882 2loss/SRqqt2/assignment_loss1.7402555794866235 1loss/FBt1/assignment_loss0.18087098399502627 0loss/FBt2/assignment_loss0.2612865524890367 /loss/FRt1/detection_loss2.8282397005817677 % _timestamp1.7857963641675236e+09 .loss/FRt2/assignment_loss4.67250372001103 2loss/SRqqt1/assignment_loss1.7128054079521133 /loss/FRt2/detection_loss3.7328454842524867 %loss/total_loss47.16544377080692 trainer/global_step2399 1loss/SRqqt2/detection_loss1.9023765123294676 .loss/FBt2/detection_loss16.05570282936096 /loss/SRqqt1/detection_loss2.52062764869096 .loss/FBt1/detection_loss7.498969263248631 epoch0 _runtime1076.990357046 _step191 K _runtime1076.990357046 _step191 .loss/FRt1/assignment_loss4.05896008840882 2loss/SRqqt2/assignment_loss1.7402555794866235 1loss/FBt1/assignment_loss0.18087098399502627 0loss/FBt2/assignment_loss0.2612865524890367 /loss/FRt1/detection_loss2.8282397005817677 % _timestamp1.7857963641675236e+09 .loss/FRt2/assignment_loss4.67250372001103 2loss/SRqqt1/assignment_loss1.7128054079521133 /loss/FRt2/detection_loss3.7328454842524867 %loss/total_loss47.16544377080692 trainer/global_step2399 1loss/SRqqt2/detection_loss1.9023765123294676 .loss/FBt2/detection_loss16.05570282936096 /loss/SRqqt1/detection_loss2.52062764869096 .loss/FBt1/detection_loss7.498969263248631 epoch0蹛(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts125.633' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent41.87766666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0kUTT(:  U network.recv189623 disk.drbd1552.out0 disk.drbd1828.in0 disk.rbd0.in0 disk.drbd1584.in0 disk.drbd2111.out0 disk.drbd2041.in0 disk.drbd1226.in0 disk.drbd2078.in0 disk.drbd1356.in0 disk.drbd2118.in0 disk.drbd2111.in0 disk.drbd1817.out0 disk.drbd1965.out0% memory_percent41.224122047424316 disk.drbd1965.in0 disk.drbd1285.out0 disk.drbd2041.out0 disk.rbd0.out0 disk.sda.in0 disk.drbd1510.out0 disk.sda2.in0 disk.drbd1510.in0 disk.drbd2118.out0 disk.sda2.out 30.171875 disk.nvme0n1.in 2.47265625 disk.drbd1673.out0 disk.drbd1938.out0$ disk./.usageGB578.3334579467773 disk.drbd1828.out0 disk.drbd2078.out0 disk.drbd1356.out0 network.sent 2.238931e+06 disk.drbd1931.out0 disk.drbd1552.in0 disk.drbd1550.in0 disk.sda.out 30.171875 disk.sda1.out0 disk.drbd1285.in0 disk.drbd1226.out0 disk.rbd1.out 4.4716796875( proc.memory.availableMB 38519.359375 disk.drbd1289.in0) proc.memory.percent17.18212366104126 cpu14.192736737341873 disk.drbd1550.out0 disk.dm-0.in 2.47265625 disk.drbd1584.out0 disk.rbd1.in0 disk.drbd1938.in0 disk.sda1.in0# disk.nvme0n1.out25.51025390625# proc.memory.rssMB 11260.4765625 proc.cpu.threads29 disk.drbd1673.in0 disk.drbd1931.in0 disk.drbd1289.out0 disk.dm-0.out25.51025390625 disk.drbd1817.in0* disk./.usagePercent32.343434516000954QB(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts98.903' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent32.967666666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0MUTV(:  ݵV proc.cpu.threads29 disk.drbd1965.out0 disk.drbd1550.out0# proc.memory.rssMB 11260.4765625 disk.drbd2111.out0 disk.drbd1552.out0 disk.rbd0.out0 disk.sda.out 30.62890625 disk.dm-0.out 25.7392578125* proc.memory.availableMB38519.09765625 cpu14.184325790463593* disk./.usagePercent32.343430249244136 disk.sda2.out 30.62890625 disk.sda1.out0" disk.nvme0n1.out 25.7392578125 disk.drbd1289.out0 disk.drbd1938.in0 disk.drbd1226.out0 disk.drbd1356.in0 disk.rbd1.out 4.5341796875 disk.drbd1673.out0 disk.drbd1938.out0 disk.drbd1285.in0 disk.sda1.in0 disk.drbd2078.out0 disk.drbd1584.in0 disk.drbd1584.out0 disk.nvme0n1.in 2.47265625 disk.drbd1285.out0# disk./.usageGB578.333381652832 disk.drbd1965.in0 disk.drbd1931.in0 disk.drbd1828.in0 disk.rbd0.in0 disk.drbd2078.in0 network.recv189689 disk.drbd1356.out0 disk.drbd2041.in0 disk.drbd1828.out0$ memory_percent41.22452139854431 disk.drbd1673.in0 disk.drbd1931.out0 disk.drbd1552.in0 disk.dm-0.in 2.47265625 disk.drbd1226.in0 disk.drbd1817.in0 disk.drbd1817.out0 disk.drbd2041.out0 disk.drbd1550.in0 network.sent 2.238997e+06 disk.drbd2118.in0 disk.rbd1.in0 disk.drbd1289.in0 disk.sda.in0) proc.memory.percent17.18212366104126 disk.sda2.in0 disk.drbd2118.out0 disk.drbd2111.in0 disk.drbd1510.in0 disk.drbd1510.out0'2- _wandbruntime1098  _runtime10981 % _timestamp1.7857963857174237e+09 _runtime1098.540395803 _step192 & lr-AdamW/pg16.311608470251464e-05 & lr-AdamW/pg26.311608470251464e-05 trainer/global_step2449jW & lr-AdamW/pg26.311608470251464e-05 trainer/global_step2449 % _timestamp1.7857963857174237e+09 _runtime1098.540395803 _step192 & lr-AdamW/pg16.311608470251464e-05jAf2- _wandbruntime1098  _runtime1098'O ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated44377411700224 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated25878403 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated16279929 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated9648047 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 2150131588312 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 2155406987776 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 _step193 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 2213989336064 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 _runtime1098.541698752 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed42063095010304 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2155494417408 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed44377249830912 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed43168098449784 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 2150217391436 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2214126301696 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed9648045 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated42063112049664 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed16279927 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated41017983900832 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated42063112049664 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated44218606467072 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed42148674 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed42163260494848 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2155494417408 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated42158332 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated16672185 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed44218501998080 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2155406987776 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated25878403 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed16279927 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current248 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current246 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed25868747 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated7024138 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 trainer/global_step2449 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed16671937 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed7023892 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816,2yO N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2155494417408 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated42158332 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated16672185 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed44218501998080 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2155406987776 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated25878403 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed16279927 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current248 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current246 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed25868747 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated7024138 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 trainer/global_step2449 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed16671937 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed7023892 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated44377411700224 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated25878403 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated16279929 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated9648047 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 2150131588312 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 2155406987776 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 _step193 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 2213989336064 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 _runtime1098.541698752 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed42063095010304 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2155494417408 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed44377249830912 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed43168098449784 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 2150217391436 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2214126301696 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed9648045 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated42063112049664 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed16279927 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated41017983900832 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated42063112049664 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated44218606467072 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed42148674 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed42163260494848 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak118871T2- _wandbruntime1098  _runtime1098eV\"&M NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed25879270 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated44236652856832 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated44395518689280 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated9651984 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 $ _timestamp1.785796386098544e+09 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed42165840 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated42175512 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2156286607360 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed42080261773312 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed42080261773312 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed42180467611648 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated42175512 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2151094864770 trainer/global_step2449 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated42080278812672 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated42180492515328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 2156286607360 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated41034724073152 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed16678727 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 2156374044160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed42165840 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated43185818937922 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated16286572 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed25879270 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated42080278812672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 2151009061534 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated44236652856832 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed16286570 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current261 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed44395358924288 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed41034707033792 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed16286570 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 _runtime1098.921916665 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 2214891312640 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121&M R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed43185716095326 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed7026745 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated44236652856832 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated44395518689280 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated9651984 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 $ _timestamp1.785796386098544e+09 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed42165840 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated42175512 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2156286607360 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed42080261773312 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed42080261773312 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed42180467611648 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated42175512 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2151094864770 trainer/global_step2449 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated42080278812672 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated42180492515328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 2156286607360 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated41034724073152 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed16678727 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 2156374044160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed42165840 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated43185818937922 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated16286572 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed25879270 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated42080278812672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 2151009061534 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated44236652856832 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed16286570 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current261 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed44395358924288 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed41034707033792 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed16286570 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 _runtime1098.921916665 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 2214891312640 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed25879270 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 E 1loss/SRqqt1/detection_loss1.5466386912818888 /loss/FBt1/detection_loss3.0865528657539265 epoch0 trainer/global_step2449 _runtime1098.923076218 _step195 0loss/FRt1/assignment_loss3.9285765609355887 1loss/FBt1/assignment_loss0.09529808980137618 1loss/FBt2/assignment_loss0.31903823271240983 .loss/FRt1/detection_loss2.870568811893463 .loss/FRt2/detection_loss3.211336460010505 %loss/total_loss39.98676127101253 /loss/FRt2/assignment_loss4.306300731352818 2loss/SRqqt1/assignment_loss1.8219305299795592 0loss/SRqqt2/detection_loss2.519495097344097 .loss/FBt2/detection_loss14.26578575372696 $ _timestamp1.785796386100243e+09 1loss/SRqqt2/assignment_loss2.015239446219944Xc 0loss/SRqqt2/detection_loss2.519495097344097 .loss/FBt2/detection_loss14.26578575372696 $ _timestamp1.785796386100243e+09 1loss/SRqqt2/assignment_loss2.015239446219944 1loss/SRqqt1/detection_loss1.5466386912818888 /loss/FBt1/detection_loss3.0865528657539265 epoch0 trainer/global_step2449 _runtime1098.923076218 _step195 0loss/FRt1/assignment_loss3.9285765609355887 1loss/FBt1/assignment_loss0.09529808980137618 1loss/FBt2/assignment_loss0.31903823271240983 .loss/FRt1/detection_loss2.870568811893463 .loss/FRt2/detection_loss3.211336460010505 %loss/total_loss39.98676127101253 /loss/FRt2/assignment_loss4.306300731352818 2loss/SRqqt1/assignment_loss1.8219305299795592+5](: U gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts123.186' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent41.062000000000005 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0V(:  U disk.drbd1938.out0 disk.sda1.in0 disk.drbd1510.in0 disk.drbd2111.in0# proc.memory.rssMB 11260.4765625) disk./.usagePercent32.34343366264959 disk.rbd1.in0 disk.sda1.out0 disk.drbd1289.out0 disk.rbd0.in0 disk.drbd1285.in0 disk.dm-0.out 25.9501953125 disk.sda2.in0 network.sent 2.355366e+06 disk.drbd1550.out0 disk.sda.in0 disk.drbd2078.in0 network.recv194949 disk.drbd1552.in0 disk.drbd1938.in0 disk.drbd2078.out0 disk.drbd1817.out0 proc.cpu.threads29 disk.drbd1289.in0) proc.memory.availableMB 38519.0546875) proc.memory.percent17.18212366104126$ disk./.usageGB578.3334426879883 disk.drbd1965.in0 disk.drbd1931.in0 disk.drbd1931.out0 disk.drbd1226.in0 disk.drbd2041.out0 disk.drbd1356.out0 disk.drbd1965.out0 disk.drbd1673.in0 disk.drbd1673.out0 disk.drbd2118.in0% memory_percent41.224586963653564 disk.drbd1828.in0 disk.drbd1510.out0 disk.drbd1584.out0" disk.nvme0n1.out 25.9501953125 disk.sda2.out 30.88671875 disk.drbd2111.out0 disk.nvme0n1.in 2.47265625 disk.drbd1356.in0 disk.rbd0.out0 disk.drbd1552.out0 disk.drbd2118.out0 disk.drbd2041.in0 disk.drbd1226.out0 disk.drbd1584.in0 cpu14.175926258423491 disk.rbd1.out 4.5341796875 disk.sda.out 30.88671875 disk.drbd1285.out0 disk.dm-0.in 2.47265625 disk.drbd1817.in0 disk.drbd1828.out0 disk.drbd1550.in0^AB2- _wandbruntime1120  _runtime1120+k% & lr-AdamW/pg26.440469402677994e-05 trainer/global_step2499 % _timestamp1.7857964077527213e+09 _runtime1120.575690968 _step196 & lr-AdamW/pg16.440469402677994e-05 & lr-AdamW/pg26.440469402677994e-05 trainer/global_step2499 % _timestamp1.7857964077527213e+09 _runtime1120.575690968 _step196 & lr-AdamW/pg16.440469402677994e-05Y2- _wandbruntime1120  _runtime1120l`GO ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2199490552832 M@DeviceStatsMonitor.on_train_baR2'tch_start/allocation.all.allocated43018484 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed26396749 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 _runtime1120.578117555 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed42921433160704 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2259320953344 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed45120836283904 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current267 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2199490552832 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2199403123200 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated45120940752896 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated42921450200064 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated44049098371776 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed43023616334848 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed26396749 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed45120836283904 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current267 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2199490552832 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2199403123200 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated45120940752896 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated42921450200064 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated44049098371776 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed43023616334848 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed26396749 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 I6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 % _timestamp1.7857964081226232e+09 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated45138987142656 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated43040848355328 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed16618720 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2200282742784 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current259 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed42938599923712 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 trainer/global_step2499 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 _step198 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed26407272 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated26416942 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed42938599923712 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 2260085020160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _runtime1120.946009498 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated45138987142656 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated43035664 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated7169942 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed43025992 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current261 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated41871732689152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed45300908471808 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed17018515 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed7169683 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed16618720 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 2200370179584 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated16618722 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated17018776 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed9848832 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed43040823451648 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed41871715649792 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated16618722 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed17018515 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed7169683 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed16618720 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 2200370179584 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated16618722 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated17018776 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed9848832 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed43040823451648 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed41871715649792 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated16618722 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 % _timestamp1.7857964081226232e+09 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated45138987142656 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated43040848355328 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed16618720 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2200282742784 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current259 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed42938599923712 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 trainer/global_step2499 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 _step198 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed26407272 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated26416942 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed42938599923712 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 2260085020160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _runtime1120.946009498 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated45138987142656 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated43035664 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated7169942 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed43025992 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current261 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated41871732689152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed45300908471808 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 R2- _wandbruntime1120  _runtime11208 /loss/FRt2/assignment_loss4.392568559646606 /loss/FRt1/assignment_loss4.107806402583455 1loss/SRqqt1/assignment_loss1.641110276667083 1loss/SRqqt2/detection_loss1.8216056081912637 _step199 1loss/FBt1/assignment_loss0.06552351462398126 0loss/FBt2/assignment_loss0.5674272478674538 /loss/FRt1/detection_loss3.2457419419011404 .loss/FRt2/detection_loss3.335478330453237 .loss/FBt1/detection_loss3.214992202759669 2loss/SRqqt2/assignment_loss1.7715419251767417 1loss/SRqqt1/detection_loss2.0714331668137618 .loss/FBt2/detection_loss9.017960917949676 &loss/total_loss35.253190094634064 epoch0 trainer/global_step2499 $ _timestamp1.785796408124345e+09 _runtime1120.947177221T 1loss/SRqqt1/detection_loss2.0714331668137618 .loss/FBt2/detection_loss9.017960917949676 &loss/total_loss35.253190094634064 epoch0 trainer/global_step2499 $ _timestamp1.785796408124345e+09 _runtime1120.947177221 /loss/FRt2/assignment_loss4.392568559646606 /loss/FRt1/assignment_loss4.107806402583455 1loss/SRqqt1/assignment_loss1.641110276667083 1loss/SRqqt2/detection_loss1.8216056081912637 _step199 1loss/FBt1/assignment_loss0.06552351462398126 0loss/FBt2/assignment_loss0.5674272478674538 /loss/FRt1/detection_loss3.2457419419011404 .loss/FRt2/detection_loss3.335478330453237 .loss/FBt1/detection_loss3.214992202759669 2loss/SRqqt2/assignment_loss1.7715419251767417'(: ضU gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts107.623' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent35.87433333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors03wS(:  U network.sent 2.356178e+06$ disk./.usageGB578.3335227966309# disk.nvme0n1.out26.24755859375 disk.sda2.in0 disk.drbd1965.in0 disk.drbd1673.in0 disk.drbd2041.out0 disk.drbd1285.in0 disk.drbd2118.in0 disk.drbd2118.out0 disk.drbd1552.in0 disk.drbd1510.out0 network.recv195081 disk.drbd1817.out0# memory_percent41.2243127822876( proc.memory.availableMB 38519.234375 disk.drbd1828.out0 disk.dm-0.in 2.47265625 disk.drbd1817.in0 disk.rbd1.in0 disk.drbd1550.in0 disk.sda.out 31.18359375 disk.drbd1584.in0) proc.memory.percent17.18212366104126 disk.drbd1356.out0 disk.drbd2111.in0 disk.drbd2111.out0 disk.drbd1965.out0 disk.drbd1552.out0 disk.drbd1285.out0 disk.drbd2078.out0 disk.drbd1584.out0 disk.nvme0n1.in 2.47265625 disk.drbd1289.out0 disk.sda2.out 31.18359375 disk.drbd1931.in0 disk.rbd0.out0 disk.sda1.in0 cpu14.168357175281713) disk./.usagePercent32.34343814274425 disk.sda.in0 proc.cpu.threads29 disk.drbd1356.in0 disk.drbd1673.out0 disk.drbd1550.out0 disk.drbd1289.in0 disk.drbd1226.in0 disk.dm-0.out26.24755859375# proc.memory.rssMB 11260.4765625 disk.drbd1931.out0 disk.drbd1938.in0 disk.drbd1226.out0 disk.drbd2078.in0 disk.rbd1.out 4.56640625 disk.drbd2041.in0 disk.drbd1828.in0 disk.rbd0.in0 disk.drbd1938.out0 disk.sda1.out0 disk.drbd1510.in0펻;(: ˮؐU gpu.0.gpu28.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts98.28' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent32.76 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0?U(:  ˮU disk.drbd2118.in0 disk.drbd2041.in0 network.sent 2.356286e+06 disk.drbd1828.out0 disk.drbd1550.out0 network.recv195189# disk.nvme0n1.out26.51318359375 disk.sda2.out 31.54296875 disk.drbd2111.out0 disk.drbd1552.in0 disk.sda.out 31.54296875 disk.drbd1584.in0 disk.drbd1817.out0 disk.drbd1550.in0 disk.drbd1289.in0 disk.drbd1289.out0 disk.rbd0.out0 disk.drbd1510.out0 disk.drbd2118.out0 disk.rbd1.in0 disk.drbd1673.in0 disk.dm-0.out26.51318359375) proc.memory.availableMB 38518.6640625# proc.memory.rssMB 11260.4765625) disk./.usagePercent32.34335856772961 disk.drbd1356.in0 disk.drbd2111.in0 disk.drbd1938.out0$ memory_percent41.22518301010132$ disk./.usageGB578.3320999145508 disk.nvme0n1.in 2.47265625 disk.drbd1828.in0 disk.drbd1510.in0 disk.drbd1226.in0 disk.drbd1931.in0 disk.drbd1552.out0 disk.drbd2078.in0 disk.drbd1938.in0 disk.sda.in0 disk.sda1.out0 disk.drbd1584.out0 disk.drbd2041.out0) proc.memory.percent17.18212366104126 proc.cpu.threads29 disk.drbd1817.in0 disk.rbd0.in0 disk.drbd1285.out0 cpu14.160572342090381 disk.sda1.in0 disk.drbd1356.out0 disk.sda2.in0 disk.rbd1.out 4.66015625 disk.drbd1965.out0 disk.drbd1226.out0 disk.drbd1285.in0 disk.drbd2078.out0 disk.drbd1965.in0 disk.drbd1673.out0 disk.drbd1931.out0 disk.dm-0.in 2.47265625a=2- _wandbruntime1142  _runtime11421^R & lr-AdamW/pg16.569330335104525e-05 & lr-AdamW/pg26.569330335104525e-05 trainer/global_step2549 % _timestamp1.7857964298328478e+09 _runtime1142.655828235 _step200| & lr-AdamW/pg16.569330335104525e-05 & lr-AdamW/pg26.569330335104525e-05 trainer/global_step2549 % _timestamp1.7857964298328478e+09 _runtime1142.655828235 _step200{2- _wandbruntime1142  _runtime1142='O KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 _runtime1142.657734458 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated16944229 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed46023170569728 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed43779771311104 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed43883972174848 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated43878636 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated26934407 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed7309521 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed42691984093472 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated7309787 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated46023275038720 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated43779788350464 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2243399258624 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated43883997078528 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated16944229 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed16944227 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 trainer/global_step2549 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 2243399258624 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2243486688256 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed10041745 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated44929995451284 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 2237994318452 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated46023275038720 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed17351266 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed46188347240448 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed43779771311104 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed46023170569728 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2243486688256 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed26924751 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130s1#O SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed10041745 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated44929995451284 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 2237994318452 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated46023275038720 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed17351266 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed46188347240448 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed43779771311104 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed46023170569728 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2243486688256 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed26924751 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 _runtime1142.657734458 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated16944229 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed46023170569728 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed43779771311104 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed43883972174848 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated43878636 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated26934407 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed7309521 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed42691984093472 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated7309787 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated46023275038720 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated43779788350464 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2243399258624 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated43883997078528 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 WJDeviceStatsMonitor.oon_train_batch_start/requested_bytes.small_pool.current85803124 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated16944229 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed16944227 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 trainer/global_step2549 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 2243399258624 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2243486688256 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0Y2- _wandbruntime1143  _runtime1143xtA_&M C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current245 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current243 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated46041321428480 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated26944944 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated43895816 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated43901204195328 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed46041216952320 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed7312309 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2238871791786 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed44947510254342 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2244366315008 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed46206459600384 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated16950872 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 2244366315008 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated46041321428480 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed43796938074112 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated10045684 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated7312552 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed43901179291648 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed17357991 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed26935274 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed26935274 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated26944944 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 trainer/global_step2549 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated43796955113472 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2244278878208 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 2238785988550 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated43895816 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated43796955113472 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 2305408878592 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 2244278878208 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed43886144 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed43796938074112 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed16950870 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed10045682 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed42708724265792 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 _runtime 1143.03704604 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360ɨk&M SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated43901204195328 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed46041216952320 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed7312309 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2238871791786 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed44947510254342 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2244366315008 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed46206459600384 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated16950872 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 2244366315008 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated46041321428480 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed43796938074112 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated10045684 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated7312552 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed43901179291648 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed17357991 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed26935274 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed26935274 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated26944944 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 trainer/global_step2549 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated43796955113472 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2244278878208 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 2238785988550 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated43895816 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated43796955113472 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 2305408878592 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 2244278878208 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed43886144 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed43796938074112 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed16950870 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed10045682 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed42708724265792 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 _runtime 1143.03704604 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current245 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current243 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated46041321428480 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated26944944 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated43895816 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160q\2- _wandbruntime1143  _runtime1143z 0loss/FRt1/assignment_loss3.8599092074802943 /loss/FRt2/assignment_loss4.379126702036176 0loss/FBt1/assignment_loss0.2999716226957389 0loss/FBt2/assignment_loss0.4832515765912831 -loss/FRt1/detection_loss2.67437873284022 epoch0 2loss/SRqqt1/assignment_loss1.6180332164679254 .loss/FRt2/detection_loss3.576713196720396 0loss/SRqqt1/detection_loss2.460953514747987 %loss/total_loss33.65911345263858 % _timestamp1.7857964302153387e+09 0loss/SRqqt2/detection_loss1.935103196817562 .loss/FBt2/detection_loss6.703672429561164 _runtime1143.038179576 2loss/SRqqt2/assignment_loss1.7303459044131968 /loss/FBt1/detection_loss3.9376541522666386 trainer/global_step2549 _step203i 2loss/SRqqt2/assignment_loss1.7303459044131968 /loss/FBt1/detection_loss3.9376541522666386 trainer/global_step2549 _step203 0loss/FRt1/assignment_loss3.8599092074802943 /loss/FRt2/assignment_loss4.379126702036176 0loss/FBt1/assignment_loss0.2999716226957389 0loss/FBt2/assignment_loss0.4832515765912831 -loss/FRt1/detection_loss2.67437873284022 epoch0 2loss/SRqqt1/assignment_loss1.6180332164679254 .loss/FRt2/detection_loss3.576713196720396 0loss/SRqqt1/detection_loss2.460953514747987 %loss/total_loss33.65911345263858 % _timestamp1.7857964302153387e+09 0loss/SRqqt2/detection_loss1.935103196817562 .loss/FBt2/detection_loss6.703672429561164 _runtime1143.038179576T(: ڮU gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts123.137' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent41.04566666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0tVhR(:  ڮU disk.drbd1817.out0) proc.memory.availableMB 38518.3828125 disk.drbd2041.in0 disk.drbd1552.out0 disk.sda.out 31.9296875 disk.sda1.out0# disk.nvme0n1.out26.79541015625 disk.drbd2111.in0 disk.drbd1965.in0 disk.drbd1965.out0 network.recv196328 disk.drbd2041.out0 disk.drbd1584.in0 disk.drbd1356.in0# proc.memory.rssMB 11260.4765625 cpu14.15256295670514 disk.drbd2078.in0 disk.drbd1584.out0 disk.drbd1817.in0 disk.drbd1931.in0 disk.drbd1552.in0 disk.drbd1289.in0 disk.drbd1289.out0 disk.rbd0.in0 disk.rbd0.out0 disk.drbd1226.out0 disk.dm-0.out26.79541015625 disk.drbd2118.in0 disk.rbd1.out 4.80859375* disk./.usagePercent32.343573825611024 disk.drbd1550.in0 disk.drbd2078.out0$ disk./.usageGB578.3359489440918 disk.sda.in0 disk.drbd1285.in0 disk.drbd1226.in0 disk.drbd1285.out0 disk.drbd1510.in0 disk.drbd2111.out0 disk.dm-0.in 2.47265625 disk.drbd1673.in0 disk.drbd1938.out0 disk.drbd1828.in0 disk.drbd1938.in0 disk.sda2.in0 proc.cpu.threads29 disk.drbd1510.out0 disk.nvme0n1.in 2.47265625 disk.drbd1356.out0# memory_percent41.2256121635437 disk.drbd1673.out0 disk.drbd1931.out0) proc.memory.percent17.18212366104126 network.sent 2.356835e+06 disk.drbd1828.out0 disk.drbd1550.out0 disk.sda2.out 31.9296875 disk.rbd1.in0 disk.sda1.in0 disk.drbd2118.out0*2- _wandbruntime1164  _runtime1164Ƒ & lr-AdamW/pg16.698191267531055e-05 & lr-AdamW/pg26.698191267531055e-05 trainer/global_step2599 $ _timestamp1.785796452164001e+09 _runtime1164.986969013 _step204 $ _timestamp1.785796452164001e+09 _runtime1164.986969013 _step204 & lr-AdamW/pg16.698191267531055e-05 & lr-AdamW/pg26.698191267531055e-05 trainer/global_step2599wDT2- _wandbruntime1164  _runtime1164 I6O [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2287482823680 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated44738788 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed27452753 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated47094050530816 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated27462409 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current243 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2287395394048 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated7452629 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2349697612288 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed43528992709472 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated10238597 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed45810789688308 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 2287395394048 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed47093897050112 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated46925609324544 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed46925504855552 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2287482823680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed44638109461504 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed17690983 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated43529009748832 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated17276379 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed44744328014848 _step205 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated44744352918528 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed17276377 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 trainer/global_step2599 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130.'O [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed43528992709472 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated10238597 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed45810789688308 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 2287395394048 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed47093897050112 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated46925609324544 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed46925504855552 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2287482823680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed44638109461504 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed17690983 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated43529009748832 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated17276379 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed44744328014848 _step205 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated44744352918528 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed17276377 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 trainer/global_step2599 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2287482823680 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated44738788 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed27452753 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated47094050530816 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated27462409 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current243 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2287395394048 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated7452629 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2349697612288 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1\.N\2- _wandbruntime1165  _runtime1165ms&M aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 2350598727680 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current234 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed7455222 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2288362450432 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated44655293263872 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 2288362450432 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated44755968 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 151376384 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed47112007386624 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2282760255294 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed44761535131648 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed45828407333850 _runtime1165.365258892 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current236 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed17697754 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed27463276 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated44655293263872 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated44761560035328 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 % _timestamp1.7857964525419438e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed17283020 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed44746296 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated7455456 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 126472704 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 2288275013632 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed44746296 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated27472946 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated44755968 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320"4&M WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed47112007386624 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2282760255294 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed44761535131648 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed45828407333850 _runtime1165.365258892 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current236 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed17697754 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed27463276 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated44655293263872 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated44761560035328 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 % _timestamp1.7857964525419438e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed17283020 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed44746296 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated7455456 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 126472704 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 2288275013632 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed44746296 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated27472946 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated44755968 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 2350598727680 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current234 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed7455222 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2288362450432 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated44655293263872 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 2288362450432 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated44755968 TFDeviceStatsML8GKonitor.on_train_batch_end/inactive_split_bytes.all.current 151376384P2- _wandbruntime1165  _runtime1165MU(X 0loss/SRqqt2/detection_loss1.796679853824346 _runtime1165.366744237 /loss/FRt2/assignment_loss4.353284751667696 % _timestamp1.7857964525438914e+09 /loss/FRt1/assignment_loss3.868734484133513 2loss/SRqqt2/assignment_loss1.8880417705027026 .loss/FRt2/detection_loss3.555870482369381 1loss/SRqqt1/detection_loss2.1845293535034194 .loss/FBt2/detection_loss6.244700616056269 %loss/total_loss31.33078540557386 epoch0 trainer/global_step2599 2loss/SRqqt1/assignment_loss1.5737728350090259 1loss/FBt1/assignment_loss0.11287812215034916 1loss/FBt2/assignment_loss0.24365673684290165 .loss/FRt1/detection_loss2.957511452553065 /loss/FBt1/detection_loss2.5511249469611936 _step207 (c .loss/FRt2/detection_loss3.555870482369381 1loss/SRqqt1/detection_loss2.1845293535034194 .loss/FBt2/detection_loss6.244700616056269 %loss/total_loss31.33078540557386 epoch0 trainer/global_step2599 2loss/SRqqt1/assignment_loss1.5737728350090259 1loss/FBt1/assignment_loss0.11287812215034916 1loss/FBt2/assignment_loss0.24365673684290165 .loss/FRt1/detection_loss2.957511452553065 /loss/FBt1/detection_loss2.5511249469611936 _step207 0loss/SRqqt2/detection_loss1.796679853824346 _runtime1165.366744237 /loss/FRt2/assignment_loss4.353284751667696 % _timestamp1.7857964525438914e+09 /loss/FRt1/assignment_loss3.868734484133513 2loss/SRqqt2/assignment_loss1.8880417705027026I&(: U gpu.0.gpu28.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts126.15' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent42.050000000000004 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0GWH(:  U disk.rbd1.out 4.80859375 disk.drbd1285.out0 disk.drbd1510.in0 disk.drbd2078.out0 disk.drbd1584.out0 disk.drbd1673.in0 disk.sda.out32.125 disk.drbd1285.in0 network.sent 2.427432e+06 disk.drbd1584.in0 disk.sda.in0* proc.memory.availableMB38518.31640625# proc.memory.rssMB 11260.4765625 proc.cpu.threads29 disk.drbd1938.in0 disk.sda1.in0 disk.drbd1226.in0 disk.drbd1226.out0 disk.drbd1965.in0 disk.drbd1673.out0 disk.drbd2041.out0 disk.drbd1552.in0) disk./.usagePercent32.34336646122972 disk.nvme0n1.in 2.47265625 disk.drbd1356.in0 disk.drbd1938.out0 disk.drbd2118.out0 disk.drbd1550.out0 disk.rbd0.out0 disk.sda2.in0 disk.drbd1817.out0 disk.drbd1965.out0 disk.drbd1931.in0 disk.drbd1828.in0 network.recv199332 disk.drbd1550.in0 disk.drbd2118.in0 disk.drbd1931.out0 disk.drbd2041.in0 disk.drbd1828.out0 disk.rbd0.in0 disk.sda2.out32.125 disk.drbd2111.in0 disk.drbd2111.out0 disk.drbd1552.out0$ disk./.usageGB578.3322410583496 disk.dm-0.in 2.47265625 disk.dm-0.out 26.994140625! disk.nvme0n1.out 26.994140625 disk.drbd1817.in0 disk.rbd1.in0 disk.sda1.out0$ memory_percent41.22571349143982 disk.drbd1356.out0) proc.memory.percent17.18212366104126 cpu14.145548581786558 disk.drbd1510.out0 disk.drbd2078.in0 disk.drbd1289.in0 disk.drbd1289.out0;+8Ϝ(: ֖U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts106.182' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent35.394000000000005 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0}YO(:  U disk.drbd2078.out0 disk.drbd2118.out0 disk.drbd2111.in0 disk.rbd1.in0 disk.drbd1673.out0 disk.drbd1552.out0 network.sent 2.430561e+06 disk.drbd1938.out0 disk.drbd1285.in0 disk.drbd1285.out0 disk.dm-0.out27.68994140625 disk.drbd1226.out0 disk.drbd1817.out0 disk.rbd1.out 4.87109375 disk.drbd2111.out0# memory_percent41.2257194519043$ disk./.usageGB578.3322410583496 disk.drbd2041.in0& proc.memory.availableMB 38518.3125 disk.drbd1510.out0 disk.dm-0.in 2.47265625 disk.drbd1673.in0 disk.drbd1550.in0 disk.sda1.in0 disk.drbd1931.in0 disk.drbd1931.out0) disk./.usagePercent32.34336646122972 disk.drbd1289.out0 disk.drbd1584.in0 disk.drbd1828.out0 disk.drbd1938.in0 disk.nvme0n1.in 2.47265625 disk.drbd1356.out0 disk.drbd1965.out0 disk.drbd2041.out0 disk.rbd0.in0 disk.rbd0.out0 disk.sda.in0 disk.drbd1356.in0 network.recv200528# disk.nvme0n1.out27.68994140625) proc.memory.percent17.18212366104126 proc.cpu.threads29# proc.memory.rssMB 11260.4765625 cpu14.138615162985065 disk.drbd1510.in0 disk.drbd2118.in0 disk.drbd1552.in0 disk.drbd1289.in0 disk.sda2.out 32.5390625 disk.drbd1817.in0 disk.drbd1965.in0 disk.sda.out 32.5390625 disk.drbd1550.out0 disk.drbd1226.in0 disk.drbd2078.in0 disk.drbd1584.out0 disk.sda2.in0 disk.drbd1828.in0 disk.sda1.out0D512- _wandbruntime1186  _runtime1186@ & lr-AdamW/pg16.827052199957586e-05 & lr-AdamW/pg26.827052199957586e-05 trainer/global_step2649 % _timestamp1.7857964739913144e+09 _runtime 1186.81424929 _step208@ _step208 & lr-AdamW/pg16.827052199957586e-05 & lr-AdamW/pg26.827052199957586e-05 trainer/global_step2649 % _timestamp1.7857964739913144e+09 _runtime 1186.81424929/yO2- _wandbruntime1186  _runtime1186bd\ܼ'O VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 _runtime1186.815475992 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated45496464651264 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed47827839141376 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated45604708758528 _step209 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed17608527 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 2325685442344 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated45598940 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated27990411 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated45496464651264 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed45496447611904 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated47999599490560 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed47827839141376 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed45604683854848 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated44366018364832 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 2325771245468 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed27980755 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed7594934 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated47827943610368 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated47827943610368 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed45496447611904 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current244 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed44366001325472 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed17608527 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2331391529472 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 2394764252160 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed47999448107008 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed46691686767816 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 126479872 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2394890732032 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2331478959104 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed18030379 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated27990411 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated46691789610300'O WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2331391529472 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 2394764252160 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed47999448107008 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed46691686767816 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 126479872 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2394890732032 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2331478959104 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed18030379 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated27990411 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated46691789610300 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 _runtime1186.815475992 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated45496464651264 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed47827839141376 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated45604708758528 _step209 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed17608527 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 2325685442344 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated45598940 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated27990411 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated45496464651264 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed45496447611904 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated47999599490560 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed47827839141376 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed45604683854848 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated44366018364832 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 2325771245468 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed27980755 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed7594934 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated47827943610368 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated47827943610368 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed45496447611904 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current244 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed44366001325472 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed17608527 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0zsG2- _wandbruntime1187  _runtime1187ܪM >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 2326562915566 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated48017718978560 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 2332271149056 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed7597701 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 141152768 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated45513631414272 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated28000948 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed17615170 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated18037327 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed10439382 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed45513614374912 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated10439384 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current242 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated17615172 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed17615170 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed45606448 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated28000948 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated44382758537152 trainer/global_step2649 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed27991278 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated45513631414272 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed45606448 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated47845990000128 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated7597943 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed18037083 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated45621915875328 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated45616120 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 _step210 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed18037083 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated45621915875328 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated45616120 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 _step210 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 2326562915566 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated48017718978560 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 2332271149056 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed7597701 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 141152768 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated45513631414272 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated28000948 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed17615170 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated18037327 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed10439382 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed45513614374912 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated10439384 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current242 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated17615172 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed17615170 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed45606448 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated28000948 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated44382758537152 trainer/global_step2649 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed27991278 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated45513631414272 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130R2- _wandbruntime1187  _runtime1187O " /loss/FRt1/assignment_loss3.951297365991693 2loss/SRqqt1/assignment_loss1.9796062495748876 2loss/SRqqt2/assignment_loss1.5864038033807863 epoch0 /loss/FRt2/assignment_loss4.186686521921402 /loss/FBt2/assignment_loss0.598650409788556 /loss/FRt1/detection_loss2.8594855857522865 .loss/FRt2/detection_loss3.117339522219621 .loss/FBt1/detection_loss3.092889667751532 .loss/FBt2/detection_loss5.183690717587104 trainer/global_step2649 1loss/SRqqt1/detection_loss2.8741094786539105 &loss/total_loss31.120421102941744 0loss/FBt1/assignment_loss0.0891500356936174 0loss/SRqqt2/detection_loss1.601111744626348 % _timestamp1.7857964743648555e+09 _runtime1187.187691627 _step211W &loss/total_loss31.120421102941744 0loss/FBt1/assignment_loss0.0891500356936174 0loss/SRqqt2/detection_loss1.601111744626348 % _timestamp1.7857964743648555e+09 _runtime1187.187691627 _step211 /loss/FRt1/assignment_loss3.951297365991693 2loss/SRqqt1/assignment_loss1.9796062495748876 2loss/SRqqt2/assignment_loss1.5864038033807863 epoch0 /loss/FRt2/assignment_loss4.186686521921402 /loss/FBt2/assignment_loss0.598650409788556 /loss/FRt1/detection_loss2.8594855857522865 .loss/FRt2/detection_loss3.117339522219621 .loss/FBt1/detection_loss3.092889667751532 .loss/FBt2/detection_loss5.183690717587104 trainer/global_step2649 1loss/SRqqt1/detection_loss2.8741094786539105>ƚ(: ʼU gpu.0.gpu0.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts94.085' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.361666666666665 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0WTT(:  V disk.sda1.out0 disk.drbd1356.out0 disk.drbd2111.out0* proc.memory.availableMB38518.53515625 proc.cpu.threads29 disk.drbd2041.in0 disk.drbd1285.in0 disk.drbd2118.out0 disk.drbd1965.in0 disk.drbd1938.in0 disk.sda.out 33.22265625 disk.nvme0n1.in 2.47265625 disk.rbd1.out 5.05859375 disk.drbd1552.in0 disk.rbd0.out0 disk.sda2.out 33.22265625# proc.memory.rssMB 11260.4765625 cpu14.132124518238918) disk./.usagePercent32.34336539454052 disk.drbd1673.in0 disk.drbd2078.in0 disk.drbd1584.in0 disk.drbd1550.in0 disk.sda1.in0 disk.drbd1226.in0 disk.drbd2111.in0 network.sent 2.430627e+06 disk.drbd1285.out0 disk.drbd1931.out0 disk.drbd1289.out0) proc.memory.percent17.18212366104126 disk.drbd1550.out0 disk.drbd1938.out0 disk.rbd0.in0 disk.sda.in0 disk.drbd1817.out0 disk.drbd1552.out0 disk.dm-0.out 28.0068359375 disk.drbd1931.in0 disk.drbd2041.out0 disk.drbd1828.out0 disk.drbd1289.in0 disk.drbd1673.out0 disk.drbd1356.in0 disk.drbd1965.out0$ disk./.usageGB578.3322219848633 disk.drbd1510.in0 disk.drbd1510.out0 disk.drbd1226.out0 disk.drbd2078.out0 disk.drbd1584.out0 disk.drbd2118.in0" disk.nvme0n1.out 28.0068359375$ memory_percent41.22537970542908 disk.drbd1828.in0 disk.dm-0.in 2.47265625 disk.sda2.in0 disk.rbd1.in0 network.recv200594 disk.drbd1817.in0`=~V2- _wandbruntime1209  _runtime1209 9 _runtime1209.232708636 _step212 & lr-AdamW/pg16.955913132384115e-05 & lr-AdamW/pg26.955913132384115e-05 trainer/global_step2699 % _timestamp1.7857964964082503e+09d!  % _timestamp1.7857964964082503e+09 _runtime1209.232708636 _step212 & lr-AdamW/pg16.955913132384115e-05 & lr-AdamW/pg26.955913132384115e-05 trainer/global_step26992(2- _wandbruntime1209  _runtime1209߂c'O ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed46465039694848 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed48904983985664 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated46459092 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated18370780 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2375387664896 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed7738238 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated17940679 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current247 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated28518413 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed28508757 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed46449434 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed46354785762304 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 _step213 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed17940677 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2375475094528 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2440072867840 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated47572686689808 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed28508757 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 2369573905852 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed17940677 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated48905137466368 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed47572583847324 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed48730173427200 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated48730277896192 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated46465064598528 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed45203009941472 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 2439944290816 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated45203026980832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 _runtime1209.232838511 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated17940679 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed18370533 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated10632297 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated48730277896192 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 2375387664896 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2375475094528 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated46354802801664 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated28518413 trainer/global_step2699 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360AO E8DeviceStatsMonitor.on_train_batch_start/active.all.freed46449434 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed46354785762304 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 _step213 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed17940677 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2375475094528 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2440072867840 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated47572686689808 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed28508757 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 2369573905852 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed17940677 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated48905137466368 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed47572583847324 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed48730173427200 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated48730277896192 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated46465064598528 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed45203009941472 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 2439944290816 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated45203026980832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 _runtime1209.232838511 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated17940679 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed18370533 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated10632297 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated48730277896192 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 2375387664896 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2375475094528 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated46354802801664 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated28518413 trainer/global_step2699@b TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed46465039694848 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed48904983985664 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated46459092 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated18370780 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2375387664896 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed7738238 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated17940679 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current247 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated28518413 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed28508757 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484b2- _wandbruntime1209  _runtime1209R&M J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 143249920 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed48923082615808 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated45219767153152 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed28519280 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated46371969564672 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed18377220 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated47590304335462 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated48748324285952 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated48748324285952 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 _step214 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated46371969564672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 2440979054080 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 _runtime1209.602461005 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated46476272 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed10636232 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2370537182310 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed17947320 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed46371952525312 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 2376267284480 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated17947322 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 168153600 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated10636234 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed46482246811648 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2376267284480 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed17947320 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed28519280 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current285 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current287 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated28528950 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed46371952525312 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated46476272 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2376354721280 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed48748219809792 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed45219750113792 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed46466600 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated48923250769408 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0g &M M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated46371969564672 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed18377220 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated47590304335462 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated48748324285952 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated48748324285952 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 _step214 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated46371969564672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 2440979054080 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 _runtime1209.602461005 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated46476272 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed10636232 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2370537182310 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed17947320 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed46371952525312 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 2376267284480 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated17947322 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 168153600 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated10636234 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed46482246811648 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2376267284480 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed17947320 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed28519280 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current285 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current287 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated28528950 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed46371952525312 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated46476272 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2376354721280 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed48748219809792 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed45219750113792 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed46466600 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated48923250769408 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 143249920 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed48923082615808 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated45219767153152 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed28519280 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed18709774 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 2413548172484 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated49810634527232 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2485214088704 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 _runtime1231.354329532 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed10829145 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated46040035596832 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated47319244 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated18272829 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2419471229952 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed47309586 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed47325395534848 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated29046415 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed18272827 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current264 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current262 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed29036759 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated7880891 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed49632507713024 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated48453583769316 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2419383800320 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed29036759 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed49810478949376 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 2413462369360O'O MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated47319244 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated18272829 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2419471229952 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed47309586 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed47325395534848 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated29046415 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed18272827 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current264 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current262 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed29036759 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated7880891 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed49632507713024 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated48453583769316 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2419383800320 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed29036759 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed49810478949376 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 2413462369360 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed48453480926832 _step217 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 trainer/global_step2749 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed49632507713024 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated29046415 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed18709774 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 2413548172484 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated49810634527232 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2485214088704 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 _runtime1231.354329532 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed10829145 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated46040035596832(E2- _wandbruntime1231  _runtime12314&M EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current246 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed18716645 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 2486116056064 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed29047282 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 2420350856704 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated18716893 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current248 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 2414339842582 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed47230290675712 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated10833084 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated7883809 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated47336424 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated47230307715072 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed29047282 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed10833082 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2414425645818 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated29056952 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated49650658571776 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated49828743611392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed18279470 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2420350856704 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed47326752 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed7883563 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 2420263419904 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2420263419904 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated18279472 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated49650658571776 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated18279472 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed49828585943552 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed47342602651648 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated46056775769152 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 trainer/global_step2749 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated47336424 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed18279470 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed47230290675712 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed49650554095616 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 _runtime1231.723335272'j? M ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 % _timestamp1.7857965189000094e+09 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 2420263419904 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2420263419904 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated18279472 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated49650658571776 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated18279472 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed49828585943552 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed47342602651648 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated46056775769152 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 trainer/global_step2749 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated47336424 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed18279470 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed47230290675712 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed49650554095616 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 _runtime1231.723335272 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current246 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 K>DeviceStatsMonitor.on_train_batch_end/inactive_splitT&,.all.freed18716645 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 2486116056064 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed29047282 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 2420350856704 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated18716893 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current248 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 2414339842582 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed47230290675712 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated10833084 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated7883809 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated47336424 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated47230307715072 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed29047282 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed10833082 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2414425645818 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated29056952 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated49650658571776 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated49828743611392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed18279470 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2420350856704 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed47326752 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed7883563 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 B _runtime1231.724596713 /loss/FRt1/assignment_loss4.135589815440931 -loss/FRt2/detection_loss3.13559025035877 1loss/SRqqt1/detection_loss3.4286882653576884 % _timestamp1.7857965189017477e+09 _step219 1loss/SRqqt1/assignment_loss1.668821958673967 1loss/SRqqt2/detection_loss1.6099911163548275 /loss/FBt1/detection_loss3.5119911537040025 epoch0 2loss/SRqqt2/assignment_loss1.6758762261317681 /loss/FBt1/assignment_loss0.573151630783741 .loss/FRt1/detection_loss2.810734211771112 /loss/FRt2/assignment_loss4.024409535684083 0loss/FBt2/assignment_loss0.4772931412823839 /loss/FBt2/detection_loss5.5228287407330106 %loss/total_loss32.57496604627629 trainer/global_step2749a(: U gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp37.0 gpu.0.powerWatts94.872' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.624000000000002 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0X(:  U disk.drbd1584.in0 network.recv210912 disk.drbd1285.out0 disk.drbd2078.out0 disk.drbd1817.out0# proc.memory.rssMB 11260.4765625) disk./.usagePercent32.34349595729912 disk.drbd2118.in0 disk.sda2.in0 disk.drbd2111.out0 disk.rbd1.out 5.1533203125 disk.drbd2041.in0 cpu14.112259911820013 disk.rbd0.in0 disk.drbd1510.in0) proc.memory.percent17.18212366104126 disk.nvme0n1.in 2.47265625 disk.drbd1817.in0 disk.drbd1289.out0 proc.cpu.threads29 disk.rbd0.out0 disk.drbd2118.out0# disk.nvme0n1.out29.30419921875 disk.drbd2041.out0 disk.sda.in0 disk.drbd1931.in0$ disk./.usageGB578.3345565795898 disk.drbd1938.out0 disk.drbd1510.out0 disk.drbd1356.in0 disk.drbd1965.out0 disk.dm-0.in 2.47265625 disk.drbd1828.in0 disk.drbd1285.in0 disk.dm-0.out29.30419921875 disk.drbd1552.out0 disk.drbd1550.out0$ memory_percent41.22515320777893 disk.sda1.out0 disk.drbd1226.in0* proc.memory.availableMB38518.68359375 disk.sda.out 34.59765625 disk.drbd1226.out0 disk.drbd2078.in0 disk.drbd1356.out0 disk.rbd1.in0 disk.drbd1828.out0 disk.sda1.in0 disk.sda2.out 34.59765625 disk.drbd1552.in0 disk.drbd1584.out0 disk.drbd1673.in0 disk.drbd1289.in0 disk.drbd1938.in0 disk.drbd2111.in0 disk.drbd1965.in0 disk.drbd1673.out0 disk.drbd1931.out0 disk.drbd1550.in0 network.sent 2.613452e+06D,a2- _wandbruntime1253  _runtime1253Ux _step220 & lr-AdamW/pg17.213634997237176e-05 & lr-AdamW/pg27.213634997237176e-05 trainer/global_step2799 % _timestamp1.7857965405738075e+09 _runtime1253.396826219 & lr-AdamW/pg27.213634997237176e-05 trainer/global_step2799 % _timestamp1.7857965405738075e+09 _runtime1253.396826219 _step220 & lr-AdamW/pg17.213634997237176e-05ص2- _wandbruntime1253  _runtime1253T,_'O WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2463379935744 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed11025995 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated50716135578112 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 trainer/global_step2799 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed48071462063104 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed50534841998848 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated46877044212832 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2463467365376 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 2457350832868 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2463467365376 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated19050052 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current238 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated48185776278528 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated50534946467840 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated29574417 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 $ _timestamp1.785796540574834e+09 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated48071479102464 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated8024055 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated18604979 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed50534841998848 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 _step221 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated11025997 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed48185751374848 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed29564761 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current240 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed19049812 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated48071479102464 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed48071462063104 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated49334480848824 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated48179396 _runtime 1253.39809206 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated29574417 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed29564761 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed49334378006340 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 2457436635992 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed50715975805952 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated18604979 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries01'O PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated18604979 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed50534841998848 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 _step221 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated11025997 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed48185751374848 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed29564761 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current240 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed19049812 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated48071479102464 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed48071462063104 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated49334480848824 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated48179396 _runtime 1253.39809206 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated29574417 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed29564761 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed49334378006340 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 2457436635992 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed50715975805952 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated18604979 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2463379935744 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed11025995 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated50716135578112 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 trainer/global_step2799 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed48071462063104 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed50534841998848 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated46877044212832 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2463467365376 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 2457350832868 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2463467365376 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated19050052 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current238 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated48185776278528 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated50534946467840 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated29574417 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 $ _timestamp1.785796540574834e+09 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated48071479102464 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated8024055=2- _wandbruntime1253  _runtime1253|BJM H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 trainer/global_step2799 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed46893767345792 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated48088645865472 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated29584954 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed29575284 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated19056819 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 2531130313216 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated46893784385152 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed19056569 MGDeviceStatsMonitor.on_train_batch_end/ina}| ctive_split.large_pool.current2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed8026637 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated48088645865472 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2458314109326 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 2464346992128 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated29584954 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 % _timestamp1.7857965409471648e+09 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated11029934 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _runtime1253.770507892 _step222 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated18611622 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed48088628826112 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed49351995651882 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 2531265174528 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated50552992857600 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2464346992128 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 2464259555328 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed48202958491648 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated49352098494478 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed18611620 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated48202983395328 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed11029932 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated48196576 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated18611622 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated50734248569856 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated50552992857600 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated48196576 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current250 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed50552888381440 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 2458228306090 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed29575284 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed18611620 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed19056569 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed8026637 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated48088645865472 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2458314109326 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 2464346992128 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated29584954 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 % _timestamp1.7857965409471648e+09 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated11029934 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _runtime1253.770507892 _step222 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated18611622 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed48088628826112 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed49351995651882 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 2531265174528 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated50552992857600 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2464346992128 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 2464259555328 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed48202958491648 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated49352098494478 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed18611620 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated48202983395328 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed11029932 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated48196576 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated18611622 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated50734248569856 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated50552992857600 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated48196576 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current250 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed50552888381440 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 2458228306090 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed29575284 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed18611620 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed50215275085848 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed48929800213504 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed8167682 trainer/global_step2849 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated51437280753664 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated49046132118528 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated47714052828832 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated51621680244736 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed47714035789472 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2507376071168 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2575548126208 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated50215377928332 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2507463500800 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 2507376071168 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated18937129 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 _step225 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed30092763 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated8167949 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 2575411160576 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated19390796 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed51621518375424 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed49029890 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 _runtime1275.965185938 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed18937127 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2507463500800 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated18937129 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed11222845 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 2501239296376 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated11222847 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 % _timestamp1.7857965631418977e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed18937127 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed51437176284672 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated48929817252864aR7'O NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2507376071168 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2575548126208 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated50215377928332 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2507463500800 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 2507376071168 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated18937129 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 _step225 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed30092763 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated8167949 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 2575411160576 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated19390796 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed51621518375424 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed49029890 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 _runtime1275.965185938 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed18937127 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2507463500800 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated18937129 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed11222845 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 2501239296376 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated11222847 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 % _timestamp1.7857965631418977e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed18937127 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed51437176284672 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated48929817252864 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed51437176284672 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed19390527 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current267 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed50215275085848 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed48929800213504 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed8167682 trainer/global_step2849 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated51437280753664 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated49046132118528 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated47714052828832 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated51621680244736 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed47714035789472 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 2502116769598 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed48946966976512 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated11226784 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed50232892731390 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated51639786349056 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated8170746 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated49063339235328 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 % _timestamp1.7857965635114262e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed18943770 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed30103286 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current263 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated48946984015872 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated30112956 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 2576312252416 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated48946984015872 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed30103286 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2502202572834 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated19397530 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated51455327143424 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed47730775961792 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2508255690752 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed51455222667264 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated18943772 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated51455327143424 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated49056728 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated18943772 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed19397267 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 _step226 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed18943770 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated47730793001152 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed49063314331648 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0g͚&M KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated49056728 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed49047056 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 2576447113728 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated8170746 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated49063339235328 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 % _timestamp1.7857965635114262e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed18943770 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed30103286 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current263 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated48946984015872 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated30112956 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 2576312252416 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated48946984015872 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed30103286 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2502202572834 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated19397530 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated51455327143424 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed47730775961792 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2508255690752 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed51455222667264 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated18943772 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated51455327143424 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated49056728 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated18943772 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed19397267 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 _step226 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed18943770 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated47730793001152 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed49063314331648 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated30112956 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed48946966976512 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed51639626584064 _runtime1276.334746559 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2508343127552 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed49047056 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 2508255690752 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current261 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 2502116769598 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed48946966976512 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated11226784 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed50232892731390 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated51639786349056 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 EDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed19269277 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed30620765 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2551372206592 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed30620765 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed49788138363904 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed52339510570496 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed19731372 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated8311905 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 trainer/global_step2899 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 2545213563008 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated49899700 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated49906487958528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed52527058540544 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated51096275007840 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed19269277 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated19269279 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated52339615039488 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2551459636224 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed49906463054848 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed52339510570496 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2551459636224 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed8311677 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated49788155403264 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 _step229 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed51096172165356 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current230 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2620719868416 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 124382720 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated11419697 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated19269279 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated19731602 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 2620595485696 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 % _timestamp1.7857965854086359e+09 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated30630421 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 2545127759884 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 149286400 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed48551044405472 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 2551372206592 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated30630421 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated52527207826944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0:'O D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed52527058540544 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated51096275007840 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed19269277 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated19269279 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated52339615039488 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2551459636224 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed49906463054848 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed52339510570496 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2551459636224 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed8311677 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated49788155403264 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 _step229 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed51096172165356 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current230 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2620719868416 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 124382720 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated11419697 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated19269279 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated19731602 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 2620595485696 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 % _timestamp1.7857965854086359e+09 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated30630421 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 2545127759884 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 149286400 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed48551044405472 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 2551372206592 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated30630421 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated52527207826944 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed49788138363904 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current228 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed49890042 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated49788155403264 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated52339615039488 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated48551061444832 _runtime1298.232083684 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed11419695 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed19269277 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed30620765 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2551372206592 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed30620765 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed49788138363904 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed52339510570496 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed19731372 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated8311905 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 trainer/global_step2899 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 2545213563008 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated49899700 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated49906487958528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed19738168 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated30640958 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed30631288 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated52357661429248 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated30640958 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed49907208 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated19275922 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 _step230 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated51113892653494 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed51113789810898 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2546091036342 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 2621502629376 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2552251826176 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed49907208 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed19275920 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed11423632 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed48567784577792 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current249 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated49916880 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed30631288 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed49805305126912 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 trainer/global_step2899 _runtime1298.612888087 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed49923670171648 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2552339262976 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated49805322166272 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated8314785 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated49916880 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 LDDeviceStatsMonitor.on_train_batch_end/inact߅4ive_split.small_pool.peak538 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated49923695075328 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated19275922 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated52545330468864 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated49805322166272 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 2552339262976 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596R&M E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated52545330468864 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated49805322166272 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 2552339262976 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated19738419 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed19738168 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated30640958 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed30631288 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated52357661429248 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated30640958 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed49907208 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated19275922 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 _step230 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated51113892653494 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed51113789810898 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2546091036342 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 2621502629376 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2552251826176 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed49907208 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed19275920 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed11423632 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed48567784577792 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current249 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated49916880 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed30631288 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed49805305126912 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 trainer/global_step2899 _runtime1298.612888087 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed49923670171648 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2552339262976 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated49805322166272 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated8314785 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated49916880 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated49923695075328 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated19275922 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 2505231362- _wandbruntime1298  _runtime1298=EO _runtime1298.614062155 .loss/FBt2/detection_loss4.967207019979304 0loss/FRt1/assignment_loss3.9285743286212287 2loss/SRqqt2/assignment_loss1.6815823288243108 0loss/FBt2/assignment_loss0.4414575516288592 /loss/FRt1/detection_loss2.7064444053297243 .loss/FRt2/detection_loss2.876606312858594 1loss/SRqqt1/detection_loss1.9425186981728308 trainer/global_step2899 _step231 /loss/FRt2/assignment_loss4.454476814765434 2loss/FBt1/assignment_loss0.029560120212836945 /loss/FBt1/detection_loss2.8353412640469906 &loss/total_loss29.337170927440155 2loss/SRqqt1/assignment_loss1.6948825846115747 1loss/SRqqt2/detection_loss1.7785194983884662 epoch0 % _timestamp1.7857965857912197e+09!  0loss/FRt1/assignment_loss3.9285743286212287 2loss/SRqqt2/assignment_loss1.6815823288243108 0loss/FBt2/assignment_loss0.4414575516288592 /loss/FRt1/detection_loss2.7064444053297243 .loss/FRt2/detection_loss2.876606312858594 1loss/SRqqt1/detection_loss1.9425186981728308 trainer/global_step2899 _step231 /loss/FRt2/assignment_loss4.454476814765434 2loss/FBt1/assignment_loss0.029560120212836945 /loss/FBt1/detection_loss2.8353412640469906 &loss/total_loss29.337170927440155 2loss/SRqqt1/assignment_loss1.6948825846115747 1loss/SRqqt2/detection_loss1.7785194983884662 epoch0 % _timestamp1.7857965857912197e+09 _runtime1298.614062155 .loss/FBt2/detection_loss4.967207019979304(: ĸU gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts127.373' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.45766666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0P{L(:  ˖V disk.sda.in0 disk.sda2.out 36.0859375 disk.drbd2041.in0) proc.memory.percent17.18212366104126 disk.drbd1289.in0 disk.drbd1938.out0 disk.dm-0.out 31.40234375 disk.drbd1226.out0 disk.rbd1.out 5.341796875 disk.nvme0n1.out 31.40234375# proc.memory.rssMB 11260.4765625 cpu14.092935669476839 disk.rbd0.out0 disk.sda.out 36.0859375 disk.drbd1510.in0 disk.drbd1828.out0 network.sent 2.708875e+06( disk./.usagePercent32.3434916905423 disk.dm-0.in 2.47265625 disk.drbd2078.out0 disk.drbd1356.out0 disk.drbd1965.out0 disk.drbd1550.in0 proc.cpu.threads29 disk.sda1.in0 disk.sda2.in0 disk.drbd1931.out0$ memory_percent41.22507572174072 network.recv216894 disk.drbd1817.out0 disk.drbd1938.in0 disk.drbd1285.out0 disk.rbd1.in0 disk.drbd1965.in0 disk.drbd2078.in0 disk.drbd2118.out0 disk.drbd1356.in0 disk.drbd2111.out0 disk.drbd1550.out0$ disk./.usageGB578.3344802856445 disk.rbd0.in0 disk.drbd1285.in0 disk.drbd1226.in0 disk.drbd2111.in0 disk.drbd1673.in0 disk.drbd1828.in0 disk.drbd1510.out0 disk.drbd2118.in0 disk.drbd1289.out0 disk.sda1.out0 disk.drbd1584.in0 disk.drbd2041.out0 disk.drbd1817.in0 disk.drbd1673.out0 disk.drbd1584.out0 disk.drbd1931.in0 disk.drbd1552.in0 disk.drbd1552.out0 disk.nvme0n1.in 2.47265625( proc.memory.availableMB 38518.734375 u0$(: U gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts94.297' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.432333333333336 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0u:< (:  V disk.drbd2041.in0 disk.drbd1552.in0 disk.sda1.in0 disk.drbd1510.out0 disk.drbd1584.in0 network.sent 2.754579e+06 disk.drbd1817.out0 disk.drbd1938.in0 disk.dm-0.in 2.47265625 disk.drbd1356.in0 disk.rbd1.out 5.435546875 cpu14.094638450306263 disk.drbd1289.in0 disk.drbd2118.in0 disk.nvme0n1.in 2.47265625$ disk./.usageGB578.3344612121582 disk.drbd1289.out0 disk.rbd0.in0 disk.drbd2118.out0 disk.nvme0n1.out31.625 proc.cpu.threads29 disk.drbd1673.in0 disk.drbd1510.in0 disk.drbd2078.out0 disk.drbd1550.in0 disk.drbd1550.out0 disk.drbd2111.out0) proc.memory.availableMB 38518.7109375 disk.drbd1965.in0 disk.drbd1965.out0 disk.rbd0.out0 disk.sda.out36.625 disk.drbd1285.in0 disk.drbd1226.in0 disk.drbd1285.out0 disk.dm-0.out31.625 disk.drbd2078.in0 disk.sda2.out36.625 disk.drbd1817.in0 disk.drbd1931.in0 disk.drbd2041.out0 disk.drbd1552.out0 disk.drbd1584.out0 disk.sda2.in0 disk.drbd2111.in0) proc.memory.percent17.18212366104126 disk.drbd1938.out0 disk.sda.in0$ memory_percent41.22511148452759# proc.memory.rssMB 11260.4765625) disk./.usagePercent32.34349062385309 disk.rbd1.in0 disk.sda1.out0 network.recv219099 disk.drbd1673.out0 disk.drbd1931.out0 disk.drbd1828.in0 disk.drbd1828.out0 disk.drbd1226.out0 disk.drbd1356.out0=2 - _wandbruntime1320  _runtime1320ٜ  & lr-AdamW/pg17.600217794516768e-05 & lr-AdamW/pg27.600217794516768e-05 trainer/global_step2949 % _timestamp1.7857966079363928e+09 _runtime1320.759351867 _step232G  & lr-AdamW/pg17.600217794516768e-05 & lr-AdamW/pg27.600217794516768e-05 trainer/global_step2949 % _timestamp1.7857966079363928e+09 _runtime1320.759351867 _step2321^2 - _wandbruntime1320  _runtime1320p5}' O G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2595368342016 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed31148767 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed53241844856320 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2665939594752 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated49388070060832 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 _runtime1320.760633245 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed11616545 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed50766818894848 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 2589102026516 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated50646493553664 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed51977069244864 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed50646476514304 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 2595368342016 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated11616547 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated53241949325312 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2595455771648 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated50646493553664 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current252 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 trainer/global_step2949 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed53241844856320 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed19601427 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated50759852 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 2589016223392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current254 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated50766843798528 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed8454954 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 % _timestamp1.7857966079373386e+09 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated19601429 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed50750194 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 _step233 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed20071499 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed50646476514304 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed31148767 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2595455771648 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated31158423 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated8455206 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated20071753 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated53432783393280 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated31158423 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed53432621523968 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 2665802629120 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated19601429 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated51977172087348 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0I" O SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed49388053021472 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed53241844856320 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed19601427 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated50759852 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 2589016223392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current254 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated50766843798528 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed8454954 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 % _timestamp1.7857966079373386e+09 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated19601429 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed50750194 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 _step233 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed20071499 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed50646476514304 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed31148767 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2595455771648 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated31158423 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated8455206 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated20071753 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated53432783393280 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated31158423 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed53432621523968 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 2665802629120 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated19601429 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated51977172087348 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 ZLDeviceStatsMonitor.on_train_batch_ystart/inactive_split_bytes.small_pool.peak 145651200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2595368342016 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed31148767 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed53241844856320 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2665939594752 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated49388070060832 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 _runtime1320.760633245 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed11616545 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed50766818894848 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 2589102026516 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated50646493553664 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed51977069244864 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed50646476514304 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 2595368342016 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated11616547 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated53241949325312 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2595455771648 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated50646493553664 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current252 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 trainer/global_step2949 IDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated50777032 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed53450723316224 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed53259891238912 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 2589893696614 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated49404810233152 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed49404793193792 $ _timestamp1.785796608308497e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed31159290 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated31168960 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated8458036 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2589979499850 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated50663660316672 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed31159290 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated53450889372672 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 2666697304576 _step234 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 141152768 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated19608072 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current248 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed8457790 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed50784026011648 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated50777032 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2596247961600 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current246 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 166056448 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed19608070 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2596335398400 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed50767360 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated53259995715072 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed20078272 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated11620484 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed51994686890406 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated19608072 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed50663643277312 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed19608070 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated20078520 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated51994789733002 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 trainer/global_step2949ϰ& M M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated50663660316672 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed31159290 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated53450889372672 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 2666697304576 _step234 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 141152768 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated19608072 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current248 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed8457790 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed50784026011648 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated50777032 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2596247961600 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current246 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 166056448 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed19608070 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2596335398400 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed50767360 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated53259995715072 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed20078272 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated11620484 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed51994686890406 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated19608072 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed50663643277312 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed19608070 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated20078520 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated51994789733002 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 trainer/global_step2949 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated53259995715072 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed50767360 _runtime1321.131897497 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated50663660316672 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated50777032 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed53450723316224 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed53259891238912 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 2589893696614 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated49404810233152 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed49404793193792 $ _timestamp1.785796608308497e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed31159290 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated31168960 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated8458036 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2589979499850 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0,2 - _wandbruntime1321  _runtime1321#  0loss/FRt1/assignment_loss3.9305918831979074 /loss/FRt2/detection_loss3.1410407924817667 1loss/SRqqt1/detection_loss1.2890861580540276 trainer/global_step2949 $ _timestamp1.785796608310503e+09 _runtime1321.133338921 1loss/FBt1/assignment_loss0.06316760769186658 0loss/FBt2/assignment_loss0.5152425548876636 /loss/FRt1/detection_loss2.8091812784312875 0loss/SRqqt2/detection_loss1.798297554217136 %loss/total_loss33.68200286781575 2loss/SRqqt1/assignment_loss1.6425806334219784 2loss/SRqqt2/assignment_loss1.8236017650471323 .loss/FBt1/detection_loss5.470409078228018 epoch0 /loss/FRt2/assignment_loss4.209528582436698 .loss/FBt2/detection_loss6.989274979720266 _step235  2loss/SRqqt1/assignment_loss1.6425806334219784 2loss/SRqqt2/assignment_loss1.8236017650471323 .loss/FBt1/detection_loss5.470409078228018 epoch0 /loss/FRt2/assignment_loss4.209528582436698 .loss/FBt2/detection_loss6.989274979720266 _step235 0loss/FRt1/assignment_loss3.9305918831979074 /loss/FRt2/detection_loss3.1410407924817667 1loss/SRqqt1/detection_loss1.2890861580540276 trainer/global_step2949 $ _timestamp1.785796608310503e+09 _runtime1321.133338921 1loss/FBt1/assignment_loss0.06316760769186658 0loss/FBt2/assignment_loss0.5152425548876636 /loss/FRt1/detection_loss2.8091812784312875 0loss/SRqqt2/detection_loss1.798297554217136 %loss/total_loss33.68200286781575+6Ȓ (: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp37.0 gpu.0.powerWatts124.984' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent41.66133333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0X (:  U disk.drbd1552.in0 disk.rbd0.in0 disk.drbd2078.in0 network.sent 2.799546e+06 disk.drbd1285.out0 disk.sda2.out 37.19140625 disk.drbd1226.in0 disk.drbd1285.in0 disk.drbd1828.out0 proc.cpu.threads29 disk.drbd1817.in0 disk.drbd2041.out0 disk.drbd1828.in0 disk.sda1.out0) proc.memory.availableMB 38518.9140625 disk.dm-0.in 2.47265625 disk.drbd1510.in0 disk.drbd1938.in0 disk.drbd2118.in0 disk.rbd1.out 5.498046875 disk.drbd1550.in0# proc.memory.rssMB 11260.4765625 disk.drbd1226.out0 disk.rbd1.in0 disk.drbd1552.out0 disk.dm-0.out32.29052734375 disk.drbd2041.in0 disk.sda.in0 cpu14.095411942738975 disk.sda.out 37.19140625% memory_percent41.224801540374756 disk.nvme0n1.in 2.47265625# disk.nvme0n1.out32.29052734375 disk.drbd2111.in0 network.recv221543 disk.drbd1931.out0) proc.memory.percent17.18212366104126 disk.drbd1356.out0 disk.drbd2111.out0 disk.drbd1965.out0 disk.drbd1673.out0 disk.drbd1931.in0 disk.drbd1510.out0 disk.sda2.in0 disk.drbd1817.out0 disk.drbd1550.out0 disk.drbd1938.out0 disk.rbd0.out0 disk.drbd1584.in0 disk.drbd1584.out0* disk./.usagePercent32.343487637123324$ disk./.usageGB578.3344078063965 disk.drbd1965.in0 disk.drbd1673.in0 disk.drbd1289.in0 disk.sda1.in0 disk.drbd1356.in0 disk.drbd2078.out0 disk.drbd2118.out0 disk.drbd1289.out0k2 - _wandbruntime1343  _runtime1343ۙR  $ lr-AdamW/pg27.7290787269433e-05 trainer/global_step2999 % _timestamp1.7857966302733335e+09 _runtime1343.096296735 _step236 $ lr-AdamW/pg17.7290787269433e-05o(  _step236 $ lr-AdamW/pg17.7290787269433e-05 $ lr-AdamW/pg27.7290787269433e-05 trainer/global_step2999 % _timestamp1.7857966302733335e+09 _runtime1343.096296735(L2 - _wandbruntime1343  _runtime13432w  O QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated51504831704064 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated54144283611136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed54338144270336 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed51627174734848 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated50225078676832 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 trainer/global_step2999 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed54144179142144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2711102306816 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed50225061637472 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 2710969535488 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 2632990490024 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2639451907072 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated19933579 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated20411349 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _step237 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2639364477440 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2639451907072 dQDe˧viceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated51627199638528 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed52857966324372 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated8597952 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 2632904686900 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated31686425 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed54144179142144 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated51620004 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated52858069166856 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated31686425 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed8597709 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 % _timestamp1.7857966302742841e+09 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed20411104 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed19933577 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed19933577 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed31676769 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed51504814664704 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed51504814664704 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated19933579 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current243 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated51504831704064 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated11813397 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed51610346 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated54144283611136 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed54144179142144 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated51620004 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated52858069166856 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated31686425 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed8597709 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 % _timestamp1.7857966302742841e+09 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed20411104 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed19933577 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed19933577 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed31676769 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed51504814664704 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed51504814664704 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated19933579 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current243 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated51504831704064 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated11813397 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed51610346 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated54144283611136 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated51504831704064 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated54144283611136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed54338144270336 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed51627174734848 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated50225078676832 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 trainer/global_step2999 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed54144179142144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2711102306816 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed50225061637472 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 2710969535488 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 2632990490024 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816o~V2 - _wandbruntime1343  _runtime1343s& M F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed52875583969914 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated19940222 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed31687292 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated20418148 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated51637184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 2633782160122 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated51637184 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed20417895 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 141152768 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated54162330000896 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2640331533824 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated19940222 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed8600563 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed50241801809792 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated51521998467072 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 166056448 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed19940220 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed31687292 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated8600814 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed51521981427712 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed11817332 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated51521998467072 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 166056448 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed19940220 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed31687292 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated8600814 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed51521981427712 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed11817332 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated51637184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 2633782160122 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated51637184 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed20417895 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 141152768 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated54162330000896 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2640331533824 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated19940222 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed8600563 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current02(2 - _wandbruntime1343  _runtime1343%E  epoch0 2loss/SRqqt2/assignment_loss1.6327768057466499 _runtime1343.470404276 /loss/FRt1/assignment_loss3.910562937259674 1loss/SRqqt1/assignment_loss1.697716182225371 1loss/FBt1/assignment_loss0.13794582220924534 1loss/SRqqt2/detection_loss1.6715725460199962 trainer/global_step2999 _step239 /loss/FRt2/assignment_loss4.234654801232474 /loss/FRt2/detection_loss2.9305304519832136 .loss/FBt1/detection_loss2.346506471599363 %loss/total_loss34.00661296672955 % _timestamp1.7857966306475613e+09 0loss/FBt2/assignment_loss0.6007431030660743 /loss/FRt1/detection_loss2.6260790941119194 1loss/SRqqt1/detection_loss1.3273260436950016 /loss/FBt2/detection_loss10.890198707580566%E  /loss/FRt1/assignment_loss3.910562937259674 1loss/SRqqt1/assignment_loss1.697716182225371 1loss/FBt1/assignment_loss0.13794582220924534 1loss/SRqqt2/detection_loss1.6715725460199962 trainer/global_step2999 _step239 /loss/FRt2/assignment_loss4.234654801232474 /loss/FRt2/detection_loss2.9305304519832136 .loss/FBt1/detection_loss2.346506471599363 %loss/total_loss34.00661296672955 % _timestamp1.7857966306475613e+09 0loss/FBt2/assignment_loss0.6007431030660743 /loss/FRt1/detection_loss2.6260790941119194 1loss/SRqqt1/detection_loss1.3273260436950016 /loss/FBt2/detection_loss10.890198707580566 epoch0 2loss/SRqqt2/assignment_loss1.6327768057466499 _runtime1343.470404276q zT (: U gpu.0.gpu1.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts127.893' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent42.631 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0`u&Q (:  V disk.sda2.in0 cpu14.092184311417528 disk.drbd2118.out0 disk.drbd1965.out0 disk.drbd1289.in0 disk.sda.in0 disk.sda.out37.46875$ disk./.usageGB578.3349533081055 disk.drbd2041.out0 disk.drbd1552.in0) disk./.usagePercent32.34351814443456 disk.drbd1356.in0 disk.drbd1931.in0 disk.dm-0.out32.52490234375 network.recv223465 disk.drbd2118.in0 disk.drbd1356.out0 disk.drbd2041.in0 disk.drbd1938.in0 disk.drbd1510.in0 disk.dm-0.in 2.47265625 network.sent 2.802846e+06 disk.sda2.out37.46875 disk.drbd1673.out0 disk.drbd1931.out0 disk.drbd1817.in0 disk.drbd1552.out0 disk.drbd1828.in0 disk.rbd0.out0 disk.drbd1510.out0 proc.cpu.threads29 disk.drbd2111.in0 disk.drbd1550.in0 disk.rbd1.in0 disk.rbd1.out 5.498046875 disk.drbd1673.in0 disk.drbd1289.out0 disk.drbd1285.in0 disk.drbd1226.in0$ memory_percent41.22489094734192) proc.memory.percent17.18212366104126# disk.nvme0n1.out32.52490234375 disk.drbd1965.in0 disk.drbd1550.out0 disk.sda1.in0 disk.drbd1226.out0 disk.drbd2078.in0* proc.memory.availableMB38518.85546875 disk.drbd2111.out0 disk.drbd2078.out0 disk.drbd1584.in0 disk.nvme0n1.in 2.47265625 disk.rbd0.in0 disk.sda1.out0# proc.memory.rssMB 11260.4765625 disk.drbd1828.out0 disk.drbd1938.out0 disk.drbd1584.out0 disk.drbd1817.out0 disk.drbd1285.out0 (: U gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp37.0 gpu.0.powerWatts94.103' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.367666666666665 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0CX (:  U disk.drbd2078.in0 disk.drbd1931.in0 disk.drbd1938.in0 disk.drbd1510.out0 disk.drbd2118.in0 disk.drbd2118.out0 disk.drbd1817.out0 disk.drbd1931.out0 disk.drbd1938.out0 disk.rbd0.in0 disk.rbd0.out0 disk.sda.in0 disk.drbd2111.out0 disk.drbd1226.out0 cpu14.08590317536737# disk.nvme0n1.out32.74951171875 disk.sda2.in0 disk.drbd1673.out0 disk.drbd1510.in0 disk.drbd1584.in0 network.sent 2.845651e+06 disk.drbd1356.out0 disk.drbd2041.out0 disk.drbd1552.in0 disk.drbd1285.out0 network.recv225608 disk.sda2.out 37.80859375 disk.drbd1673.in0 disk.drbd1285.in0% memory_percent41.224902868270874 disk.nvme0n1.in 2.47265625* proc.memory.availableMB38518.84765625 disk.rbd1.in0 disk.drbd2041.in0 disk.drbd1828.in0 disk.sda1.out0 disk.drbd2078.out0# proc.memory.rssMB 11260.4765625 proc.cpu.threads29 disk.drbd2111.in0 disk.drbd1965.out0 disk.drbd1552.out0 disk.drbd1550.out0 disk.drbd1226.in0) disk./.usagePercent32.34351921112377 disk.dm-0.out32.74951171875 disk.drbd1550.in0 disk.drbd1289.in0 disk.drbd1584.out0 disk.rbd1.out 5.5302734375 disk.sda.out 37.80859375 disk.drbd1356.in0$ disk./.usageGB578.3349723815918 disk.drbd1965.in0 disk.drbd1289.out0 disk.sda1.in0 disk.dm-0.in 2.47265625) proc.memory.percent17.18212366104126 disk.drbd1817.in0 disk.drbd1828.out0 2 - _wandbruntime1365  _runtime1365 l  _step240 & lr-AdamW/pg17.857939659369829e-05 & lr-AdamW/pg27.857939659369829e-05 trainer/global_step3049 % _timestamp1.7857966528118021e+09 _runtime1365.634775051 /  & lr-AdamW/pg17.857939659369829e-05 & lr-AdamW/pg27.857939659369829e-05 trainer/global_step3049 % _timestamp1.7857966528118021e+09 _runtime1365.634775051 _step240M2 - _wandbruntime1365  _runtime1365G' O E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated20750580 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed32204771 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed20265727 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current253 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated55046617896960 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated53738966246364 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2683448042496 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 % _timestamp1.7857966528127728e+09 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated32214427 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed8740082 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed53738863403880 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 2676793150408 _runtime1365.636040118 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed20265727 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated32214427 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated52363169854464 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed52470498 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 2676878953532 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed55046513427968 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated12010247 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated55243786065408 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed51062070253472 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 _step241 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 2756091524096 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated52487555478528 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed52363152815104 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed55243622098944 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed52487530574848 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed52363152815104 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current251 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2756230586880 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 trainer/global_step3049 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2683360612864 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 2756091524096 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated52487555478528 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed52363152815104 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed55243622098944 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed52487530574848 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed52363152815104 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current251 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2756230586880 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 trainer/global_step3049 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2683360612864 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed32204771 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed20265727 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current253 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated55046617896960 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated53738966246364 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2683448042496 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 % _timestamp1.7857966528127728e+09 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated32214427 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed8740082 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed53738863403880 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 2676793150408 _runtime1365.636040118 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed20265727 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated32214427 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated52363169854464 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed52470498 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 2676878953532 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed55046513427968 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated12010247 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated55243786065408 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed51062070253472 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 _step241 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 IDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated52497336 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed55064559810560 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2684240232448 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed52487664 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed12014182 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated8743150 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current260 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated20272372 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed52380319578112 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated53756583892018 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_po|L6ol.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated52380336617472 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed52380319578112 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed32215294 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 % _timestamp1.7857966531939485e+09 _step242 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated32224964 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed20757072 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 2677670623630 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated52380336617472 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated52497336 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed32215294 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated52504762595328 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 2684327669248 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated20272372 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated55261897009152 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 trainer/global_step3049 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated55064664286720 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2684327669248 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed51078810425792 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed20272370 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated12014184 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 2757134413824 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed8742890 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed53756481049422 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed55261733049856 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 2756995358208 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2677756426866 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed52487664 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated52497336 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed55064559810560 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2684240232448 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed52487664 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed12014182 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated8743150 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current260 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated20272372 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed52380319578112 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated53756583892018 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated52380336617472 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed52380319578112 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed32215294 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 % _timestamp1.7857966531939485e+09 _step242 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated32224964 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current262 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed20757072 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 2677670623630 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated52380336617472 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated52497336 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed32215294 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated52504762595328 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 2684327669248 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated20272372 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated55261897009152 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 trainer/global_step3049 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated55064664286720 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2684327669248 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed51078810425792 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed20272370 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated12014184 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 2757134413824 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed8742890 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed53756481049422D82 - _wandbruntime1366  _runtime1366b<  2loss/SRqqt1/assignment_loss1.6498876016025674 2loss/SRqqt2/assignment_loss1.5235231399536133 1loss/FBt1/assignment_loss0.10104880557048546 1loss/SRqqt2/detection_loss1.4804785914932916 .loss/FRt2/detection_loss2.739272267758092 1loss/SRqqt1/detection_loss1.9490372114958663 $ _timestamp1.785796653195586e+09 _runtime 1366.01841138 .loss/FBt2/detection_loss3.660115122795105 &loss/total_loss29.718265381488393 epoch0 0loss/FRt1/assignment_loss3.9489341348409654 /loss/FRt2/assignment_loss4.269379190251797 1loss/FBt2/assignment_loss0.40647055341729094 .loss/FBt1/detection_loss4.887849702517159 .loss/FRt1/detection_loss3.102269059792161 trainer/global_step3049 _step243[G  2loss/SRqqt1/assignment_loss1.6498876016025674 2loss/SRqqt2/assignment_loss1.5235231399536133 1loss/FBt1/assignment_loss0.10104880557048546 1loss/SRqqt2/detection_loss1.4804785914932916 .loss/FRt2/detection_loss2.739272267758092 1loss/SRqqt1/detection_loss1.9490372114958663 $ _timestamp1.785796653195586e+09 _runtime 1366.01841138 .loss/FBt2/detection_loss3.660115122795105 &loss/total_loss29.718265381488393 epoch0 0loss/FRt1/assignment_loss3.9489341348409654 /loss/FRt2/assignment_loss4.269379190251797 1loss/FBt2/assignment_loss0.40647055341729094 .loss/FBt1/detection_loss4.887849702517159 .loss/FRt1/detection_loss3.102269059792161 trainer/global_step3049 _step243,f (: U gpu.0.gpu24.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts125.616' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent41.872 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0 r;V (:  U disk.drbd1550.out0 disk.drbd1931.out0 disk.drbd1552.in0 proc.cpu.threads29 disk.drbd2078.out0 disk.drbd2041.in0 disk.drbd1289.in0 disk.drbd1673.in0 disk.drbd1931.in0) proc.memory.percent17.18212366104126 network.recv225806 disk.sda1.out0 disk.drbd2041.out0 disk.drbd1289.out0 disk.drbd1938.in0# disk.nvme0n1.out33.01513671875 disk.drbd1828.out0# proc.memory.rssMB 11260.4765625 disk.drbd1584.out0 disk.drbd1817.in0 disk.drbd1285.in0 disk.drbd1552.out0 disk.drbd1550.in0 disk.rbd0.out0 disk.dm-0.out33.01513671875 disk.drbd1356.in0 disk.drbd2111.in0 disk.drbd1817.out0 disk.rbd0.in0 disk.sda1.in0 disk.drbd1226.out0 disk.drbd2078.in0 disk.drbd2111.out0$ disk./.usageGB578.3392753601074 disk.sda.out 38.3515625 disk.drbd1510.in0 disk.drbd2118.in0 disk.sda2.in0 disk.sda.in0 disk.drbd1584.in0 disk.rbd1.in0 disk.drbd1965.in0 network.sent 2.848491e+06 disk.drbd1938.out0 disk.dm-0.in 2.47265625 disk.nvme0n1.in 2.47265625* proc.memory.availableMB38518.81640625 disk.drbd1285.out0 disk.drbd1356.out0 disk.drbd1965.out0 disk.drbd1828.in0) disk./.usagePercent32.34375985620825 disk.drbd1510.out0 disk.drbd2118.out0 disk.sda2.out 38.3515625 disk.drbd1673.out0 disk.drbd1226.in0 disk.rbd1.out 5.6240234375% memory_percent41.224950551986694 cpu14.080436832482226W2 - _wandbruntime1387  _runtime1387h9  % lr-AdamW/pg17.98680059179636e-05 % lr-AdamW/pg27.98680059179636e-05 trainer/global_step3099 % _timestamp1.7857966748211765e+09 _runtime1387.645696721 _step2446@_L  % lr-AdamW/pg27.98680059179636e-05 trainer/global_step3099 % _timestamp1.7857966748211765e+09 _runtime1387.645696721 _step244 % lr-AdamW/pg17.98680059179636e-05ɣ2 - _wandbruntime1387  _runtime1387[' O @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 _runtime1387.645796933 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed12207095 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2727356748288 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed55948847713792 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed53221490965504 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2727444177920 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 124382720 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated20597879 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated21089616 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated56149326756352 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed8882282 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 149286400 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2727444177920 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current237 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed55948847713792 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 2801291055104 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated53221508004864 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated8882519 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed54619760483388 % _timestamp1.7857966748218634e+09 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed20597877 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated32742429 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed53221490965504 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed53347886414848 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed51899078869472 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated53340308 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 trainer/global_step3099 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated53347911318528 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current239 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed20597877 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2801415437824 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated20597879 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed54619760483388 % _timestamp1.7857966748218634e+09 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed20597877 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated32742429 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed53221490965504 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed53347886414848 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed51899078869472 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 tB#UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated53340308 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 trainer/global_step3099 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated53347911318528 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current239 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed20597877 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2801415437824 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated20597879 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2727444177920 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current237 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed55948847713792 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 2801291055104 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated53221508004864 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated8882519 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 \2 - _wandbruntime1388  _runtime1388 & M F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2721644890374 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 # _timestamp1.78579667519208e+09 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated12211034 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated54637480971526 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated8885254 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed56167286258176 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 2728236367872 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed54637378128930 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 trainer/global_step3099 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated53238674767872 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed55966894096384 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed20604520 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated32752966 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed53238657728512 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed21096050 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated55966998572544 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed53347816 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated32752966 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 2728323804672 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 2802192726528 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current236 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed53365093531648 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed51915819041792 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated20604522 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated53238674767872 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 _step246 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated53357488 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed20604520 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2728236367872 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated56167443926016 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 2802325490688 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0<& M SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated8885254 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed56167286258176 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 2728236367872 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed54637378128930 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 trainer/global_step3099 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated53238674767872 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed55966894096384 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed20604520 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated32752966 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed53238657728512 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed21096050 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated55966998572544 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed53347816 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated32752966 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 2728323804672 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 2802192726528 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current236 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed53365093531648 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed51915819041792 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated20604522 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated53238674767872 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 _step246 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated53357488 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed20604520 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2728236367872 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated56167443926016 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 2802325490688 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2721644890374 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 # _timestamp1.78579667519208e+09 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated12211034 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated54637480971526 2 - _wandbruntime1388  _runtime1388   &loss/total_loss34.600433255373986 _step247 2loss/SRqqt1/assignment_loss1.6907252555247396 2loss/FBt1/assignment_loss0.030875506319161004 0loss/FBt2/assignment_loss0.5075977604104386 .loss/FBt2/detection_loss8.804839750130972 _runtime1388.016546442 0loss/FRt1/assignment_loss3.8533232015432772 1loss/SRqqt1/detection_loss1.4412472968449583 0loss/FRt2/assignment_loss4.3808880440176345 2loss/SRqqt2/assignment_loss1.7516267532912584 .loss/FRt1/detection_loss2.589282180845123 /loss/FRt2/detection_loss2.8638682759379686 .loss/FBt1/detection_loss5.128425852989319 epoch0 trainer/global_step3099 % _timestamp1.7857966751937253e+09 1loss/SRqqt2/detection_loss1.5577333775191393e1   0loss/FRt1/assignment_loss3.8533232015432772 1loss/SRqqt1/detection_loss1.4412472968449583 0loss/FRt2/assignment_loss4.3808880440176345 2loss/SRqqt2/assignment_loss1.7516267532912584 .loss/FRt1/detection_loss2.589282180845123 /loss/FRt2/detection_loss2.8638682759379686 .loss/FBt1/detection_loss5.128425852989319 epoch0 trainer/global_step3099 % _timestamp1.7857966751937253e+09 1loss/SRqqt2/detection_loss1.5577333775191393 &loss/total_loss34.600433255373986 _step247 2loss/SRqqt1/assignment_loss1.6907252555247396 2loss/FBt1/assignment_loss0.030875506319161004 0loss/FBt2/assignment_loss0.5075977604104386 .loss/FBt2/detection_loss8.804839750130972 _runtime1388.016546442.I͐ (: ʰU gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts127.851' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent42.617 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0H (:  ʰU disk.drbd1289.out0 disk.sda.out 38.7578125 disk.sda1.out0 disk.drbd2078.in0$ memory_percent41.22503399848938 disk.sda1.in0 network.sent 2.916692e+06 network.recv229569 disk.drbd1550.out0 disk.rbd1.out 5.6865234375$ disk./.usageGB578.3432159423828 disk.drbd2041.out0 disk.drbd1226.in0 disk.drbd1356.out0 disk.sda2.out 38.7578125* proc.memory.availableMB38518.76171875 disk.drbd1673.in0 disk.drbd1552.in0 disk.drbd1938.out0 disk.drbd1285.out0 disk.drbd1356.in0 disk.drbd1817.out0 disk.drbd1965.out0 disk.drbd1828.out0 disk.drbd1938.in0 disk.drbd1226.out0 disk.drbd1584.out0 disk.drbd1828.in0 disk.drbd1931.in0 disk.drbd1552.out0 disk.drbd1550.in0 disk.sda2.in0# proc.memory.rssMB 11260.4765625 disk.drbd1584.in0 disk.drbd2118.out0 disk.drbd2041.in0 disk.drbd1510.in0) proc.memory.percent17.18212366104126 cpu14.075155930049581 disk.drbd1285.in0 disk.drbd2118.in0 disk.nvme0n1.out 33.30859375 disk.sda.in0 disk.drbd1289.in0 disk.rbd1.in0 disk.drbd1673.out0 disk.rbd0.in0 disk.drbd1817.in0 disk.drbd1931.out0 disk.rbd0.out0 disk.drbd1510.out0 proc.cpu.threads29 disk.dm-0.in 2.47265625 disk.dm-0.out 33.30859375 disk.nvme0n1.in 2.47265625 disk.drbd1965.in0 disk.drbd2078.out0 disk.drbd2111.in0 disk.drbd2111.out0) disk./.usagePercent32.343980v|23419784xtf)2 - _wandbruntime1409  _runtime1409  % lr-AdamW/pg18.11566152422289e-05 % lr-AdamW/pg28.11566152422289e-05 trainer/global_step3149 % _timestamp1.7857966970864317e+09 _runtime1409.909415682 _step248y  _runtime1409.909415682 _step248 % lr-AdamW/pg18.11566152422289e-05 % lr-AdamW/pg28.11566152422289e-05 trainer/global_step3149 % _timestamp1.7857966970864317e+092 - _wandbruntime1409  _runtime1409(o' O ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated12403947 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed33260775 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed56851181999616 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated54079846155264 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 2764655880548 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed20930027 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed56851181999616 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 _step249 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated9024918 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 166063616 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated54079846155264 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2771352883712 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated54208267158528 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2846598219264 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated55500760405380 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed54190802 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current260 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed54208242254848 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 2846457059328 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated54200460 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated52736104524832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated21428865 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 $ _timestamp1.785796697087429e+09 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated57054865377792 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated33270431 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2771440313344 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated20930029 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 _runtime1409.913654645 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed9024660 trainer/global_step3149 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated56851286468608 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed54079829115904 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed33260775ߚ]Q' O QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current258 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed56851181999616 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated54079846155264 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 2764655880548 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed20930027 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed56851181999616 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 _step249 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated9024918 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 166063616 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated54079846155264 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2771352883712 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated54208267158528 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2846598219264 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated55500760405380 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed54190802 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current260 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed54208242254848 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 2846457059328 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated54200460 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated52736104524832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated21428865 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 $ _timestamp1.785796697087429e+09 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated57054865377792 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated33270431 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2771440313344 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated20930029 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 _runtime1409.913654645 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed9024660 trainer/global_step3149 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated56851286468608 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed54079829115904 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed33260775 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated12403947 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed33260775 IDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed21435382 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 2772232503296 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed33271298 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated9027756 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 2772319940096 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 trainer/global_step3149 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed54096995878912 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed9027500 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed33271298 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed57072811979264 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 2765447550646 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2772319940096 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current258 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated54097012918272 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2772232503296 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated55518378051034 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2765533353882 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated20936672 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 _step250 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed54225449371648 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated56869332858368 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated33280968 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 $ _timestamp1.785796697478966e+09 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated54217640 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated54097012918272 _runtime1410.302355927 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated54225474275328 >6DeviceStatsMonitor.on{6K_train_batch_end/num_device_alloc251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130/& M VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated12407884 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated56869332858368 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed56869228382208 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated20936672 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 _step250 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed54225449371648 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated56869332858368 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated33280968 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 $ _timestamp1.785796697478966e+09 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated54217640 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated54097012918272 _runtime1410.302355927 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated54225474275328 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current256 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed52752827657792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated52752844697152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed21435382 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 2772232503296 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed33271298 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated9027756 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 2772319940096 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 trainer/global_step3149 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed54096995878912 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed9027500 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed33271298 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed57072811979264 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 2765447550646 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2772319940096 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current258 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated54097012918272 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2772232503296 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated55518378051034 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2765533353882 ODeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed57753516285440 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed57753516285440 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed55050954 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated53573113140832 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 % _timestamp1.7857967190496075e+09 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed54938167266304 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated57960459395072 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated57753620754432 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current265 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed56381554642404 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2815436448768 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated21262179 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 _runtime1431.872862577 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 2808544344056 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 141159936 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed54938167266304 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated9167593 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 2808458540932 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2815436448768 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 166063616 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2815349019136 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated54938184305664 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated55060612 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed21262177 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 2815349019136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 trainer/global_step3199 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed33788777 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed21768123 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 _step253 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated57753620754432 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed57960293331456 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124Ʀp  O LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 2815349019136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 trainer/global_step3199 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed33788777 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed21768123 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 _step253 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated57753620754432 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed57960293331456 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated12600797 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed57753516285440 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed57753516285440 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed55050954 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated53573113140832 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 % _timestamp1.7857967190496075e+09 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed54938167266304 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated57960459395072 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated57753620754432 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current265 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed56381554642404 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2815436448768 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated21262179 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 _runtime1431.872862577 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 2808544344056 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 =QG>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 141159936 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed54938167266304 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated9167593 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 2808458540932 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2815436448768 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 166063616 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2815349019136 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated54938184305664 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated55060612 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed21262177 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0]]2 - _wandbruntime1432  _runtime1432wa& M R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed57771562668032 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed55085805211648 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated53589853313152 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed53589836273792 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated55077792 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current240 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed55068120 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed21268820 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed57978408804352 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated55085830115328 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2809421817390 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed12604732 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 _step254 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated12604734 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 % _timestamp1.7857967194192615e+09 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed21774877 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated57771667144192 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed33799300 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed9170145 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated33808970 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed33799300 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed54955334029312 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated55077792 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated57978562277888 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 2892732162560 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated56399275130542 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 trainer/global_step3199 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed21268820 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed54955334029312 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 2816228638720 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated54955351068672 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current238 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated21268822 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated33808970 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated57771667144192 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 2816316075520 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated9170383 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 2809336014154 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 _runtime1432.242682707 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 2892603592704 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed55068120 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 1044761609T& M S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated57771667144192 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed33799300 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed9170145 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated33808970 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed33799300 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed54955334029312 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated55077792 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated57978562277888 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 2892732162560 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated56399275130542 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 trainer/global_step3199 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed21268820 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed54955334029312 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 2816228638720 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated54955351068672 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current238 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated21268822 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated33808970 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated57771667144192 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 2816316075520 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated9170383 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 2809336014154 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 _runtime1432.242682707 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 2892603592704 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed55068120 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed57771562668032 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed55085805211648 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated53589853313152 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed53589836273792 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated55077792 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current240 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed55068120 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed21268820 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed57978408804352 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated55085830115328 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2809421817390 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed12604732 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 _step254 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated12604734 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 % _timestamp1.7857967194192615e+09 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed217748772 - _wandbruntime1432  _runtime1432=x7  /loss/FRt2/assignment_loss3.824981553480029 .loss/FRt2/detection_loss2.990525437751785 .loss/FBt2/detection_loss9.305389285087585 .loss/FRt1/detection_loss2.612117208692492 0loss/SRqqt1/detection_loss1.546607988176833 %loss/total_loss33.32707094997536 % _timestamp1.7857967194210835e+09 _step255 2loss/SRqqt1/assignment_loss1.7588314283977855 2loss/SRqqt2/assignment_loss1.7224324261912933 0loss/FBt1/assignment_loss0.1334536915383604 0loss/FBt2/assignment_loss0.5197465523877847 /loss/FBt1/detection_loss2.9564745917944073 /loss/FRt1/assignment_loss4.319021830753404 1loss/SRqqt2/detection_loss1.6374889557235934 epoch0 trainer/global_step3199 _runtime1432.2439122425  .loss/FRt1/detection_loss2.612117208692492 0loss/SRqqt1/detection_loss1.546607988176833 %loss/total_loss33.32707094997536 % _timestamp1.7857967194210835e+09 _step255 2loss/SRqqt1/assignment_loss1.7588314283977855 2loss/SRqqt2/assignment_loss1.7224324261912933 0loss/FBt1/assignment_loss0.1334536915383604 0loss/FBt2/assignment_loss0.5197465523877847 /loss/FBt1/detection_loss2.9564745917944073 /loss/FRt1/assignment_loss4.319021830753404 1loss/SRqqt2/detection_loss1.6374889557235934 epoch0 trainer/global_step3199 _runtime1432.243912242 /loss/FRt2/assignment_loss3.824981553480029 .loss/FRt2/detection_loss2.990525437751785 .loss/FBt2/detection_loss9.305389285087585/=*d (: V gpu.0.gpu37.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts99.071' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent33.02366666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0sSO (:  W disk.nvme0n1.in 2.47265625 disk.drbd1673.in0 disk.drbd1510.out0 disk.drbd2078.in0# proc.memory.rssMB 11260.4765625 disk.drbd1552.out0 cpu14.059601669013306 network.recv233945 disk.rbd1.out 5.8984375 disk.drbd1673.out0 disk.drbd2041.out0 disk.drbd1510.in0 disk.drbd1584.in0 disk.sda2.out 39.78515625 disk.drbd1965.in0 disk.drbd1931.out0 disk.drbd1552.in0) disk./.usagePercent32.34395570034614 disk.drbd1938.in0 disk.sda1.in0 disk.drbd1817.out0 disk.dm-0.in 2.47265625 disk.drbd2118.in0 disk.drbd1356.out0 disk.drbd2111.in0) proc.memory.percent17.18212366104126 disk.drbd2041.in0 disk.dm-0.out 35.14453125* proc.memory.availableMB38517.82421875 disk.drbd1965.out0 disk.drbd1289.in0 disk.rbd0.in0 disk.drbd1817.in0 disk.rbd1.in0 disk.drbd1931.in0 disk.drbd1584.out0 disk.drbd2111.out0$ memory_percent41.22646450996399 disk.drbd1550.in0 disk.sda.in0 disk.drbd1285.in0 disk.drbd1356.in0 disk.drbd1828.in0 disk.drbd1828.out0 disk.drbd1289.out0 disk.sda.out 39.78515625 disk.drbd1226.in0 disk.drbd2118.out0 disk.drbd1550.out0 disk.nvme0n1.out 35.14453125 disk.sda2.in0 network.sent 2.967317e+06 disk.drbd1938.out0 disk.sda1.out0 disk.drbd1226.out0 disk.drbd2078.out0 proc.cpu.threads29$ disk./.usageGB578.3427772521973 disk.rbd0.out0 disk.drbd1285.out0q12 - _wandbruntime1454  _runtime1454D0  trainer/global_step3249 % _timestamp1.7857967413172624e+09 _runtime1454.140227853 _step256 & lr-AdamW/pg18.373383389075951e-05 & lr-AdamW/pg28.373383389075951e-05`  _step256 & lr-AdamW/pg18.373383389075951e-05 & lr-AdamW/pg28.373383389075951e-05 trainer/global_step3249 % _timestamp1.7857967413172624e+09 _runtime1454.140227853y2 - _wandbruntime1454  _runtime1454Z 6u O UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated22107297 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2937010570240 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated21594329 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated21594329 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated34326435 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed34316779 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current258 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed58655850571264 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed55796505416704 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed55796505416704 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed55928953934848 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 % _timestamp1.7857967413182123e+09 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed21594327 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 _step257 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed9309394 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 trainer/global_step3249 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed58655850571264 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed12797645 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed21594327 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated9309650 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 2852432807564 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 _runtime1454.141459281 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2859432584192 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed22107039 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed58865829636608 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_byt6G es.small_pool.peak 145651200 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2859432584192 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated12797647 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed55911106 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated58655955040256 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated58865989408768 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated54410121756832 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated55920764 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated58655955040256 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 2936875701760 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated57262554564396 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed54410104717472' O cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2937010570240 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated21594329 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated21594329 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated34326435 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed34316779 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current258 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed58655850571264 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed55796505416704 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed55796505416704 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed55928953934848 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 % _timestamp1.7857967413182123e+09 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed21594327 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 _step257 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed9309394 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 trainer/global_step3249 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed58655850571264 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed12797645 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed21594327 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated9309650 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 2852432807564 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 _runtime1454.141459281 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2859432584192 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed22107039 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed58865829636608 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2859432584192 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated12797647 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed55911106 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated58655955040256 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated58865989408768 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated54410121756832 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated55920764 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated58655955040256 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 2936875701760 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated57262554564396 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed54410104717472 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated22107297 C2 - _wandbruntime1454  _runtime1454& M L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2860224774144 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed22113795 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed57280069367454 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated34336972 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed55946161051648 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 2937786945536 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed54426844889792 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated21600972 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed55813672179712 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed34327302 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated55937944 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed58883947997184 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed21600970 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2860312210944 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 2860312210944 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated58884097276416 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated54426861929152 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 124375552 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated55937944 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated58674001430016 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed55813672179712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current246 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated12801584 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated9312457 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 2937911321088 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2853310280898 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated55813689219072 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed34327302 $ _timestamp1.785796741683878e+09 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated22114041 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _step258 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated57280172210050 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 2853224477662 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856*8/# M H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed55813672179712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current246 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated12801584 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated9312457 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 2937911321088 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2853310280898 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated55813689219072 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed34327302 $ _timestamp1.785796741683878e+09 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated22114041 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _step258 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated57280172210050 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 2853224477662 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2860224774144 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed22113795 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed57280069367454 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated34336972 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed55946161051648 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 2937786945536 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed54426844889792 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated21600972 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed55813672179712 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed34327302 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated55937944 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed58883947997184 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed21600970 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2860312210944 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 2860312210944 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated58884097276416 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated54426861929152 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 124375552 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated55937944 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated58674001430016 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0O2 - _wandbruntime1454  _runtime1454ă  0loss/FBt2/assignment_loss0.6574183659894126 1loss/SRqqt1/detection_loss1.3960896685106885 _runtime1454.508333405 0loss/FRt2/assignment_loss3.4987472534179687 2loss/SRqqt2/assignment_loss1.7004584995763643 1loss/FBt1/assignment_loss0.14598221003149275 0loss/SRqqt2/detection_loss1.362367971563212 %loss/total_loss32.50624156568359 .loss/FRt2/detection_loss2.588003772497177 .loss/FBt2/detection_loss5.302594545073069 epoch0 % _timestamp1.7857967416854994e+09 _step259 /loss/FRt1/detection_loss2.9393971240383454 .loss/FBt1/detection_loss7.079126669311248 trainer/global_step3249 /loss/FRt1/assignment_loss4.186818021467363 2loss/SRqqt1/assignment_loss1.6492374642072498  .loss/FRt2/detection_loss2.588003772497177 .loss/FBt2/detection_loss5.302594545073069 epoch0 % _timestamp1.7857967416854994e+09 _step259 /loss/FRt1/detection_loss2.9393971240383454 .loss/FBt1/detection_loss7.079126669311248 trainer/global_step3249 /loss/FRt1/assignment_loss4.186818021467363 2loss/SRqqt1/assignment_loss1.6492374642072498 0loss/FBt2/assignment_loss0.6574183659894126 1loss/SRqqt1/detection_loss1.3960896685106885 _runtime1454.508333405 0loss/FRt2/assignment_loss3.4987472534179687 2loss/SRqqt2/assignment_loss1.7004584995763643 1loss/FBt1/assignment_loss0.14598221003149275 0loss/SRqqt2/detection_loss1.362367971563212 %loss/total_loss32.50624156568359u (: ϼU gpu.0.gpu41.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts95.183' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent31.72766666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0M.S (:  U) disk./.usagePercent32.34403207529317 disk.drbd1938.in0 disk.drbd1552.out0 disk.drbd1550.out0 disk.rbd0.out0 disk.drbd2078.out0 disk.drbd2118.out0# disk.nvme0n1.out36.10205078125 disk.rbd1.in0 proc.cpu.threads29 disk.dm-0.out36.10205078125 disk.drbd1965.in0 cpu14.054533425160338 disk.drbd1931.out0 disk.drbd2041.in0$ memory_percent41.22774004936218 disk.drbd1550.in0 disk.drbd1285.in0 disk.drbd1285.out0 disk.drbd1584.in0 disk.drbd1828.out0 disk.drbd1938.out0 disk.drbd1584.out0 disk.drbd1817.out0 disk.drbd1552.in0 disk.rbd0.in0 disk.drbd1226.out0 network.sent 3.010912e+06 disk.drbd1828.in0 disk.drbd1289.in0 disk.sda1.out0 disk.nvme0n1.in 2.47265625) proc.memory.percent17.18212366104126 disk.drbd1289.out0 disk.sda2.in0 disk.drbd1673.in0 disk.drbd1673.out0 disk.dm-0.in 2.47265625 network.recv236431 disk.drbd1965.out0* proc.memory.availableMB38516.98828125 disk.sda.out 40.2109375 disk.drbd1226.in0 disk.drbd1817.in0$ disk./.usageGB578.3441429138184 disk.drbd2078.in0 disk.drbd1356.out0 disk.sda2.out 40.2109375 disk.drbd2111.out0 disk.sda1.in0 disk.drbd1510.in0 disk.drbd2118.in0 disk.drbd1931.in0 disk.drbd2111.in0 disk.rbd1.out 5.9609375 disk.drbd2041.out0 disk.sda.in0 disk.drbd1510.out0 disk.drbd1356.in0# proc.memory.rssMB 11260.4765625qv (: U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts119.985' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent39.995 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0}BJ (:  U disk.drbd1552.out0 disk.drbd1550.in0 cpu14.049363663614693 disk.drbd1285.in0 disk.drbd1510.out0 disk.dm-0.in 2.47265625 disk.drbd1584.out0 disk.drbd1356.out0 disk.rbd1.out 5.9609375 disk.rbd0.in0 disk.sda.out40.625 disk.drbd2078.out0 disk.drbd1965.in0 disk.drbd1938.out0 disk.sda.in0# disk./.usageGB578.344181060791 disk.drbd1226.out0 disk.drbd2118.in0 disk.drbd1356.in0 disk.drbd1931.in0 disk.drbd2111.in0 disk.drbd1828.out0* proc.memory.availableMB38517.55859375) proc.memory.percent17.18212366104126# disk.nvme0n1.out36.47021484375 disk.rbd1.in0 disk.drbd1673.in0 disk.drbd1817.in0 disk.drbd2041.in0 disk.drbd1289.in0# proc.memory.rssMB 11260.4765625 disk.drbd1285.out0 disk.drbd1673.out0 disk.dm-0.out36.47021484375 disk.drbd1550.out0$ memory_percent41.22686982154846 network.sent 3.058044e+06 disk.nvme0n1.in 2.47265625 disk.sda2.in0 disk.drbd1931.out0 disk.drbd2041.out0 network.recv238654 disk.drbd1817.out0 disk.drbd1552.in0 disk.drbd1510.in0 disk.drbd2118.out0 disk.drbd1938.in0 disk.rbd0.out0 disk.sda1.in0 disk.drbd1226.in0 disk.drbd2111.out0 disk.drbd1965.out0 disk.drbd1289.out0 disk.sda1.out0 proc.cpu.threads29) disk./.usagePercent32.34403420867158 disk.drbd2078.in0 disk.drbd1584.in0 disk.drbd1828.in0 disk.sda2.out40.625z2 - _wandbruntime1476  _runtime1476Y   trainer/global_step3299 % _timestamp1.7857967635903013e+09 _runtime 1476.4132579 _step260 & lr-AdamW/pg18.502244321502481e-05 & lr-AdamW/pg28.502244321502481e-05DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed59558184857088 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated56654860606464 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated56780916 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed55247113333472 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated22447471 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed34844781 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2903428719616 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated59771522332672 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated58143451643904?G' O XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated12994497 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 2982056979968 $ _timestamp1.785796763591223e+09 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed59558184857088 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated56654860606464 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated56780916 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed55247113333472 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated22447471 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed34844781 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2903428719616 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated59771522332672 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated58143451643904 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed56771258 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed21926477 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed56654843567104 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 _runtime1476.414461369 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2903428719616 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 2896235467948 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 trainer/global_step3299 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed34844781 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current251 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current253 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed22447218 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 2982187654144 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2903341289984 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated34854437 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed12994495 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated55263870545152 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed34855304 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2904220909568 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated56798096 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2897198744406 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated34864974 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed22453944 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated22454194 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed56788424 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current250 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 2982960027648 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed12998432 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current248 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed55263853505792 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated56806541795328 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated58161069289558 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated59576335715840 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed21933120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated34864974 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated56672027369472 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 VCDeviceStatsMonitor.on_train_batch_end/allocated_by6xtes.all.allocated59576335715840 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed56672010330112 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed59576231239680 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed21933120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed56672010330112 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 _runtime1476.784111563 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 2904220909568 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed56788424 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed59789476919296 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed56806516891648 trainer/global_step3299 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2904308346368 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed34855304 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated21933122 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 2897112941170 _step262 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed56806516891648 trainer/global_step3299 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2904308346368 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed34855304 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated21933122 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 2897112941170 _step262 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated55263870545152 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed34855304 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2904220909568 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated56798096 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2897198744406 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated34864974 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed22453944 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated22454194 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed56788424 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current250 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 2982960027648 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed12998432 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current248 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed55263853505792 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated56806541795328 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated58161069289558 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated59576335715840 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed21933120 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated34864974 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated56672027369472 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated59576335715840 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed56672010330112 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed59576231239680 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed21933120 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed56672010330112 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 _runtime1476.784111563 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 2904220909568 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed56788424m032 - _wandbruntime1476  _runtime1476   2loss/SRqqt1/assignment_loss1.6235011851609642 .loss/FRt1/detection_loss2.478611347480462 %loss/total_loss25.81747713126812 /loss/FBt1/detection_loss1.6683658055767578 /loss/FRt2/assignment_loss3.933874487876892 1loss/SRqqt2/assignment_loss1.921371721502008 0loss/FBt2/assignment_loss0.5546428418510914 .loss/FRt2/detection_loss3.033770926663133 1loss/SRqqt2/detection_loss1.4170592628419274 trainer/global_step3299 % _timestamp1.7857967639624455e+09 .loss/FBt2/detection_loss3.489422231912613 epoch0 _runtime1476.785280541 _step263 /loss/FRt1/assignment_loss4.048911832846128 1loss/FBt1/assignment_loss0.10039446253527827 1loss/SRqqt1/detection_loss1.5475510250208613V  /loss/FRt2/assignment_loss3.933874487876892 1loss/SRqqt2/assignment_loss1.921371721502008 0loss/FBt2/assignment_loss0.5546428418510914 .loss/FRt2/detection_loss3.033770926663133 1loss/SRqqt2/detection_loss1.4170592628419274 trainer/global_step3299 % _timestamp1.7857967639624455e+09 .loss/FBt2/detection_loss3.489422231912613 epoch0 _runtime1476.785280541 _step263 /loss/FRt1/assignment_loss4.048911832846128 1loss/FBt1/assignment_loss0.10039446253527827 1loss/SRqqt1/detection_loss1.5475510250208613 2loss/SRqqt1/assignment_loss1.6235011851609642 .loss/FRt1/detection_loss2.478611347480462 %loss/total_loss25.81747713126812 /loss/FBt1/detection_loss1.6683658055767578!b (: U gpu.0.gpu28.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts94.173' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.391000000000002 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0UdV (:  U disk.drbd1510.in0 disk.dm-0.in 2.47265625 disk.drbd1965.out0 disk.drbd1673.in0 disk.drbd2118.out0 disk.drbd1356.out0) proc.memory.percent17.18212366104126 disk.sda.out 40.94140625 disk.drbd1356.in0 disk.sda2.out 40.94140625 disk.rbd0.out0 disk.drbd1285.in0 disk.drbd1931.in0 disk.sda1.out0 disk.drbd2111.out0 disk.drbd1931.out0 disk.drbd2041.out0 network.recv238894 disk.drbd1938.out0 disk.drbd1226.out0 disk.drbd1584.in0 disk.drbd2041.in0 disk.drbd1552.out0 disk.drbd2078.out0 network.sent 3.060911e+06# disk./.usageGB578.344181060791 disk.rbd0.in0 disk.sda1.in0 disk.rbd1.in0 disk.drbd1938.in0 cpu14.044692686378852 disk.sda2.in0 disk.drbd1817.in0 disk.rbd1.out 5.9931640625 disk.drbd1289.out0 disk.drbd1285.out0 disk.drbd1552.in0 disk.drbd1550.in0 disk.sda.in0 disk.drbd2118.in0 disk.drbd1673.out0) disk./.usagePercent32.34403420867158 disk.drbd1510.out0 disk.drbd1226.in0 disk.drbd1817.out0# proc.memory.rssMB 11260.4765625 proc.cpu.threads29 disk.drbd2111.in0 disk.drbd1828.in0 disk.drbd1828.out0) proc.memory.availableMB 38517.5234375 disk.drbd1550.out0 disk.drbd2078.in0 disk.drbd1584.out0 disk.nvme0n1.in 2.47265625 disk.drbd1965.in0$ memory_percent41.22692346572876 disk.dm-0.out38.50146484375 disk.drbd1289.in0# disk.nvme0n1.out38.501464843752 - _wandbruntime1498  _runtime1498@Q  & lr-AdamW/pg18.631105253929012e-05 & lr-AdamW/pg28.631105253929012e-05 trainer/global_step3349 $ _timestamp1.785796785216273e+09 _runtime1498.040760206 _step264v  & lr-AdamW/pg18.631105253929012e-05 & lr-AdamW/pg28.631105253929012e-05 trainer/global_step3349 $ _timestamp1.785796785216273e+09 _runtime1498.040760206 _step264-2 - _wandbruntime1498  _runtime1498 %F' O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed22258627 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3027371813888 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 2940123931456 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated22258629 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3027234848256 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed60460519142912 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated13191347 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated59024348723412 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated56084138988832 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed57513181717504 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated60677062332416 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed35372783 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated60460623611904 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated9595710 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated35382439 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2947424855040 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed60460519142912 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 2947337425408 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed35372783 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed22786803 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current254 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed60676900463104 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed56084121949472 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2947424855040 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated57641068 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current252 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2947337425408 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated22258629 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated57513198756864 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed59024245880928 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed57513181717504 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 trainer/global_step3349 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed13191345 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed57631410 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed57649665614848 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 _step265 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024A O JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed13191345 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed57631410 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed57649665614848 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 _step265 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed22258627 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3027371813888 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 2940123931456 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated22258629 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3027234848256 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed60460519142912 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated13191347 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated59024348723412 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated56084138988832 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed57513181717504 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated60677062332416 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed35372783 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated60460623611904 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated9595710 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated35382439 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2947424855040 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed60460519142912 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 2947337425408 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed35372783 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed22786803 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current254 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed60676900463104 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed56084121949472 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 2947424855040 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated57641068 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current252 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 2947337425408 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated22258629 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated57513198756864 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed59024245880928 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed57513181717504 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 trainer/global_step3349۲2 - _wandbruntime1498  _runtime1498d& M OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3028136241152 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed22265270 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 2941001404678 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed57530348480512 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed57530348480512 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated59041966369066 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed22793555 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated60695166640640 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3028269005312 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated57658248 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 _runtime1498.409661969 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated22265272 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed9598273 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current257 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated57658248 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2948304481792 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed35383306 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 trainer/global_step3349 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 2948304481792 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated22265272 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated35392976 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2948217044992 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed22265270 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated13195284 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 2948217044992 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 $ _timestamp1.785796785586302e+09 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated57530365519872 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current255 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed56100862121792 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated57666897635328 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2941087207914 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated60478670001664 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated9598528 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed60695008972800 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed59041863526470 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated22793812 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated57658248 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2948304481792 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed35383306 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 trainer/global_step3349 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 2948304481792 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated22265272 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated35392976 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2948217044992 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed22265270 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated13195284 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 2948217044992 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 $ _timestamp1.785796785586302e+09 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated57530365519872 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current255 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed56100862121792 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated57666897635328 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2941087207914 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated60478670001664 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated9598528 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed60695008972800 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed59041863526470 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated22793812 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed22265270 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 2941001404678 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed57530348480512 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed57530348480512 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated59041966369066 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed22793555 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated60695166640640 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3028269005312 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated57658248 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 _runtime1498.409661969 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated22265272 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed95982732 - _wandbruntime1498  _runtime14985R  %loss/total_loss26.17746564945955 % _timestamp1.7857967855880275e+09 _step267 2loss/SRqqt1/assignment_loss1.5703719164643968 /loss/FRt2/detection_loss2.8427610454134276 epoch0 trainer/global_step3349 1loss/SRqqt1/detection_loss1.5064139331197086 /loss/FBt2/detection_loss3.8451341152191163 /loss/FRt1/assignment_loss4.212918259203434 .loss/FRt2/assignment_loss3.72243212285589 1loss/FBt1/assignment_loss0.11918739220058533 0loss/FBt2/assignment_loss0.4537481756415218 /loss/FRt1/detection_loss2.5798696748291454 1loss/SRqqt2/detection_loss1.5716943793243843 .loss/FBt1/detection_loss2.021925006447801 _runtime1498.410855637 2loss/SRqqt2/assignment_loss1.7310096287401393,  _step267 2loss/SRqqt1/assignment_loss1.5703719164643968 /loss/FRt2/detection_loss2.8427610454134276 epoch0 trainer/global_step3349 1loss/SRqqt1/detection_loss1.5064139331197086 /loss/FBt2/detection_loss3.8451341152191163 /loss/FRt1/assignment_loss4.212918259203434 .loss/FRt2/assignment_loss3.72243212285589 1loss/FBt1/assignment_loss0.11918739220058533 0loss/FBt2/assignment_loss0.4537481756415218 /loss/FRt1/detection_loss2.5798696748291454 1loss/SRqqt2/detection_loss1.5716943793243843 .loss/FBt1/detection_loss2.021925006447801 _runtime1498.410855637 2loss/SRqqt2/assignment_loss1.7310096287401393 %loss/total_loss26.17746564945955 % _timestamp1.7857967855880275e+09t (: ɎU gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts123.69' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent41.23 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0$AS (:  U# proc.memory.rssMB 11260.4765625 disk.drbd1938.out0 disk.dm-0.in 2.47265625 disk.drbd2078.out0 disk.drbd1584.in0 network.recv241378 disk.drbd1552.in0 disk.drbd1552.out0 disk.sda.out 41.37890625 disk.drbd1510.out0 disk.dm-0.out 39.1728515625 disk.drbd2111.out0 disk.sda1.in0 disk.drbd1550.out0 disk.rbd0.in0* disk./.usagePercent32.344044448887935 disk.drbd1931.out0 disk.drbd1550.in0 disk.drbd1285.in0 disk.drbd1817.out0 proc.cpu.threads29 disk.drbd1289.in0 disk.drbd1226.out0" disk.nvme0n1.out 39.1728515625 disk.drbd1965.out0 disk.sda.in0 disk.drbd1285.out0 disk.drbd2078.in0) proc.memory.percent17.18212366104126 disk.drbd1673.out0 cpu14.040279982239003 disk.drbd2041.in0 disk.rbd1.in0 disk.rbd1.out 6.1728515625 disk.sda1.out0 disk.drbd1510.in0 disk.nvme0n1.in 2.47265625 disk.drbd1289.out0 disk.drbd2111.in0 disk.sda2.out 41.37890625 disk.drbd1356.in0 network.sent 3.106014e+06 disk.drbd1965.in0 disk.drbd1356.out0 disk.drbd1817.in0 disk.drbd1828.in0 disk.drbd1584.out0 disk.drbd2041.out0 disk.drbd1828.out0 disk.rbd0.out0 disk.drbd1226.in0 disk.sda2.in0$ disk./.usageGB578.3443641662598 disk.drbd1673.in0 disk.drbd1931.in0 disk.drbd2118.in0 disk.drbd2118.out0$ memory_percent41.22762680053711 disk.drbd1938.in0& proc.memory.availableMB 38517.0625?뀐 (: ±U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts121.227' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent40.409 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0ѶU (:  ±U disk.nvme0n1.in 2.47265625 disk.rbd1.out 6.2353515625 disk.drbd1673.out0 disk.drbd1938.in0 disk.sda2.in0 disk.drbd1828.in0 disk.rbd0.out0 disk.drbd1285.in0 disk.drbd1226.in0 disk.drbd2078.out0 disk.drbd1817.out0$ disk./.usageGB578.3443069458008 disk.drbd1289.in0 disk.drbd1510.out0 disk.drbd1356.in0 disk.sda2.out 42.03515625 disk.drbd2111.in0 disk.drbd1965.in0) proc.memory.percent17.18212366104126* disk./.usagePercent32.344041248820325 disk.rbd0.in0 disk.sda1.in0 disk.drbd1285.out0 disk.rbd1.in0 disk.drbd1965.out0 disk.drbd1552.in0 disk.drbd1550.out0 disk.drbd1673.in0# proc.memory.rssMB 11260.4765625 disk.drbd1510.in0 disk.drbd2041.in0 proc.cpu.threads29 disk.drbd1584.in0# disk.nvme0n1.out40.04248046875 disk.sda.out 42.03515625 disk.drbd2118.out0 disk.drbd1817.in0 cpu14.039181332313747 disk.drbd1289.out0 disk.sda1.out0 disk.drbd1828.out0 network.sent 3.10608e+06 disk.drbd1938.out0 disk.drbd2111.out0 disk.drbd1931.in0 disk.drbd1552.out0 disk.dm-0.in 2.47265625 disk.drbd2078.in0 disk.drbd1931.out0( proc.memory.availableMB 38517.234375 network.recv241444 disk.drbd1226.out0# memory_percent41.2273645401001 disk.dm-0.out40.04248046875 disk.drbd2118.in0 disk.drbd1356.out0 disk.drbd2041.out0 disk.drbd1550.in0 disk.sda.in0 disk.drbd1584.out0i2 - _wandbruntime1520  _runtime1520 ˭  & lr-AdamW/pg18.759966186355543e-05 & lr-AdamW/pg28.759966186355543e-05 trainer/global_step3399 % _timestamp1.7857968079643219e+09 _runtime1520.787286117 _step268N@R  _runtime1520.787286117 _step268 & lr-AdamW/pg18.759966186355543e-05 & lr-AdamW/pg28.759966186355543e-05 trainer/global_step3399 % _timestamp1.7857968079643219e+094%2 - _wandbruntime1520  _runtime1520)W O WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 H>Device#d&StatsMonitor.on_train_batch_start/active.small_pool.peak11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed23126492 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated35910441 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated35910441 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current259 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed61362853428736 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated13388197 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed61582406608896 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed22590777 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed13388195 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 trainer/global_step3399 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated58510046358528 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated59905245802920 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed58491562 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated58501220 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed61362853428736 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 $ _timestamp1.785796807965284e+09 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed58371519867904 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed58510021454848 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 2984012394964 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated61362957897728 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated58371536907264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated23126753 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2991420990464 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed58371519867904 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3072517925376 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3072385154048 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated22590779 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed22590777 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current261f' O TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated35910441 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current259 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed61362853428736 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated13388197 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed61582406608896 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed22590777 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed13388195 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 trainer/global_step3399 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated58510046358528 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated59905245802920 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed58491562 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated58501220 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed61362853428736 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 $ _timestamp1.785796807965284e+09 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed58371519867904 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed58510021454848 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 2984012394964 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated61362957897728 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated58371536907264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated23126753 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 2991420990464 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed58371519867904 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3072517925376 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3072385154048 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated22590779 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed22590777 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current261 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed23126492 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated35910441 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130/2 - _wandbruntime1521  _runtime15211:& M NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated22597422 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 trainer/global_step3399 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed35911308 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 _runtime1521.158987208 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated35920978 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2992213180416 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 _step270 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated58527253475328 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 $ _timestamp1.785796808335593e+09 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed58388686630912 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current255 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed59922760605978 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated13392134 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed61380899811328 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed22597420 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current253 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed9741119 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed35911308 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed22597420 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 2992300617216 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated58518400 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated9741372 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed56937870737792 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated22597422 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2992300617216 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated35920978 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed23133251 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated58388703670272 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed13392132 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated61381004287488 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated58388703670272 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated58518400 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed58508728 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated56937887777152 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated23133506 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3073418393600 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2984975671422 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated61381004287488 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed58388686630912 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed61600516298240 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed58527228571648 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 27682406402  M \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 2992300617216 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated58518400 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated9741372 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed56937870737792 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated22597422 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 2992300617216 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated35920978 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed231332X}h51 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated58388703670272 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed13392132 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated61381004287488 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated58388703670272 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated58518400 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed58508728 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated56937887777152 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated23133506 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3073418393600 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 2984975671422 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated61381004287488 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed58388686630912 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed61600516298240 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed58527228571648 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated22597422 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 trainer/global_step3399 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed35911308 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 _runtime1521.158987208 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated35920978 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 2992213180416 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 _step270 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated58527253475328 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 $ _timestamp1.785796808335593e+09 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed58388686630912 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current255 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed59922760605978 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated13392134 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed61380899811328 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed22597420 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current253 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed9741119 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed35911308 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed22597420 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0E=2 - _wandbruntime1521  _runtime1521i  1loss/SRqqt2/detection_loss1.7193743050437564 %loss/total_loss26.86571319385837 epoch0 _runtime1521.160346288 _step271 /loss/FRt1/assignment_loss4.040878267849193 /loss/FBt2/assignment_loss0.717492854271768 .loss/FRt2/detection_loss2.579471519510997 1loss/SRqqt1/detection_loss1.2470736355371301 trainer/global_step3399 2loss/SRqqt1/assignment_loss1.6960529431700706 2loss/SRqqt2/assignment_loss1.7981310665361152 1loss/FBt1/assignment_loss0.17660900526225748 /loss/FRt1/detection_loss2.7800376516931196 /loss/FBt1/detection_loss2.1146921783003823 % _timestamp1.7857968083375368e+09 /loss/FRt2/assignment_loss3.963206171989441 .loss/FBt2/detection_loss4.032693594694138@}  /loss/FRt2/assignment_loss3.963206171989441 .loss/FBt2/detection_loss4.032693594694138 1loss/SRqqt2/detection_loss1.7193743050437564 %loss/total_loss26.86571319385837 epoch0 _runtime1521.160346288 _step271 /loss/FRt1/assignment_loss4.040878267849193 /loss/FBt2/assignment_loss0.717492854271768 .loss/FRt2/detection_loss2.579471519510997 1loss/SRqqt1/detection_loss1.2470736355371301 trainer/global_step3399 2loss/SRqqt1/assignment_loss1.6960529431700706 2loss/SRqqt2/assignment_loss1.7981310665361152 1loss/FBt1/assignment_loss0.17660900526225748 /loss/FRt1/detection_loss2.7800376516931196 /loss/FBt1/detection_loss2.1146921783003823 % _timestamp1.7857968083375368e+09QV| (: ѱU gpu.0.gpu31.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts102.867' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent34.289 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0^uW (:  ѱU* disk./.usagePercent32.344043382198734% memory_percent41.227078437805176# proc.memory.rssMB 11260.4765625 disk.drbd2041.in0 disk.drbd1584.in0 disk.rbd1.in0 disk.drbd1828.in0 disk.dm-0.in 2.47265625 disk.nvme0n1.in 2.47265625 disk.drbd1817.in0 disk.drbd1550.in0 disk.sda.in0 disk.drbd1510.out0 disk.drbd1226.in0 cpu14.040299565462357 network.sent 3.15357e+06 disk.sda2.out 42.33203125 disk.drbd2111.out0 disk.drbd1938.out0 disk.drbd1510.in0 disk.drbd1584.out0# disk.nvme0n1.out40.28466796875 disk.drbd1965.in0 disk.drbd1673.in0 disk.drbd1552.in0 disk.rbd0.in0 disk.drbd1817.out0 disk.dm-0.out40.28466796875 disk.drbd2078.out0 disk.drbd1965.out0 disk.drbd1289.out0 disk.rbd0.out0 disk.sda1.out0 disk.sda2.in0 disk.drbd1552.out0 disk.drbd1356.out0 disk.drbd1828.out0 disk.drbd1938.in0 disk.sda.out 42.33203125 disk.drbd2041.out0 disk.drbd1289.in0 disk.drbd1226.out0 disk.drbd1931.in0 disk.drbd1285.out0 disk.drbd2118.out0 network.recv243924 disk.rbd1.out 6.2353515625 disk.drbd1673.out0$ disk./.usageGB578.3443450927734 disk.sda1.in0 disk.drbd1285.in0 disk.drbd2078.in0 disk.drbd1356.in0) proc.memory.percent17.18212366104126 proc.cpu.threads29 disk.drbd2111.in0 disk.drbd1931.out0 disk.drbd1550.out0 disk.drbd2118.in0( proc.memory.availableMB 38517.421875t2 - _wandbruntime1542  _runtime1542a<  _runtime1542.803241253 _step272 & lr-AdamW/pg18.888827118782073e-05 & lr-AdamW/pg28.888827118782073e-05 trainer/global_step3449 $ _timestamp1.785796829980278e+09l  & lr-AdamW/pg18.888827118782073e-05 & lr-AdamW/pg28.888827118782073e-05 trainer/global_step3449 $ _timestamp1.785796829980278e+09 _runtime1542.803241253 _step272k 2 - _wandbruntime1542  _runtime1542L2' O QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated36438443 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed9881297 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated13585047 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated60786142882428 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3035329696256 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed62265187714560 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3035417125888 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed22922927 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current248 trainer/global_step3449 _step273 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed23466342 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated22922929 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed60786040039944 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed36428787 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed59370377294848 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3117698098176 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed57758139181472 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed62265187714560 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed36428787 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 $ _timestamp1.785796829981245e+09 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 3027986661596 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed13585045 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed62487942621696 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed59351714 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated59229875057664 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated23466592 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated62488100296704 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated59370402198528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated59361372 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated59229875057664 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated22922929 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated9881545 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated57758156220832 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated62265292183552 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3035329696256 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed59229858018304 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed59229858018304 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated62265292183552 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0' O G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current250 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated36438443 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed22922927 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3117565326848 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3027900858472 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 _runtime1542.804553705 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 3035417125888 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed62265187714560 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3035417125888 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed22922927 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current248 trainer/global_step3449 _step273 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed23466342 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated22922929 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed60786040039944 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed36428787 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed59370377294848 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3117698098176 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed57758139181472 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed62265187714560 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed36428787 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 $ _timestamp1.785796829981245e+09 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 3027986661596 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed13585045 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed62487942621696 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed59351714 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated59229875057664 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated23466592 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated62488100296704 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated59370402198528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated59361372 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated59229875057664 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated22922929 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated9881545 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated57758156220832 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated62265292183552 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3035329696256 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed59229858018304 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed59229858018304 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated62265292183552 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated36438443 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed9881297 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated13585047 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated60786142882428 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3035329696256 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 IDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3118464870400 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 % _timestamp1.7857968303503098e+09 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed59368880 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed22929570 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 XFDeviceStatsMonitor.on_train_batch_end/alloc&"ated_bytes.small_pool.freed 3036209315840 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated22929572 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated62506209047040 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed62506049282048 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed59247024781312 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current252 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 trainer/global_step3449 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated59378552 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated62283338573312 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated36448980 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated59247041820672 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated59378552 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed59368880 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated36448980 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated9884378 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 _step274 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3036209315840 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed57774879353792 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated62283338573312 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3028864134930 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated13588984 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed36439310 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3036296752640 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed23473108 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed59387584411648 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current254 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed36439310 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3118599731712 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated22929572 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed22929570 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed23473108 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed59387584411648 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current254 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed36439310 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3118599731712 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated22929572 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed22929570 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3118464870400 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 % _timestamp1.7857968303503098e+09 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed59368880 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed22929570 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 3036209315840 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated22929572 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated62506209047040 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed62506049282048 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed59247024781312 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current252 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 trainer/global_step3449 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated59378552 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated62283338573312 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated36448980 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated59247041820672 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated59378552 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed59368880 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated36448980 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated9884378 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 _step274 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3036209315840 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed57774879353792 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated62283338573312 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0S஍2 - _wandbruntime1543  _runtime1543%wV  1loss/FBt1/assignment_loss0.10530450834266958 /loss/FRt1/detection_loss2.5818253251695142 /loss/FRt2/detection_loss2.7210234657152377 0loss/FBt2/assignment_loss0.6318853950271239 1loss/SRqqt2/detection_loss1.5540616334262907 &loss/total_loss23.301390169816713 1loss/SRqqt1/detection_loss0.9906191396216747 _runtime1543.174896368 _step275 trainer/global_step3449 /loss/FBt2/detection_loss1.5451586429889386 epoch0 % _timestamp1.7857968303520553e+09 /loss/FBt1/detection_loss1.8282571572003297 /loss/FRt1/assignment_loss3.982728363312397 /loss/FRt2/assignment_loss3.849615287424913 1loss/SRqqt1/assignment_loss1.932540194131434 0loss/SRqqt2/assignment_loss1.57837105745619cB  _runtime1543.174896368 _step275 trainer/global_step3449 /loss/FBt2/detection_loss1.5451586429889386 epoch0 % _timestamp1.7857968303520553e+09 /loss/FBt1/detection_loss1.8282571572003297 /loss/FRt1/assignment_loss3.982728363312397 /loss/FRt2/assignment_loss3.849615287424913 1loss/SRqqt1/assignment_loss1.932540194131434 0loss/SRqqt2/assignment_loss1.57837105745619 1loss/FBt1/assignment_loss0.10530450834266958 /loss/FRt1/detection_loss2.5818253251695142 /loss/FRt2/detection_loss2.7210234657152377 0loss/FBt2/assignment_loss0.6318853950271239 1loss/SRqqt2/detection_loss1.5540616334262907 &loss/total_loss23.301390169816713 1loss/SRqqt1/detection_loss0.99061913962167475ٛ (: U gpu.0.gpu41.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts130.007' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent43.33566666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0 U (:  U disk.drbd1226.out0 disk.drbd1817.in0 disk.rbd1.in0 disk.drbd1552.out0 disk.drbd1289.in0 proc.cpu.threads29 disk.drbd1552.in0 network.recv246287* disk./.usagePercent32.344044448887935 disk.drbd2041.in0 disk.drbd1938.in0 disk.drbd1510.in0 disk.drbd1584.out0 disk.drbd2118.in0 disk.drbd1356.out0 disk.sda2.in0 disk.drbd1817.out0 disk.drbd2041.out0 disk.drbd1289.out0 disk.sda.out 42.6953125 disk.drbd1285.in0 network.sent 3.198595e+06 disk.sda.in0 disk.drbd1673.in0 disk.sda1.out0 disk.drbd2078.out0 disk.drbd1828.in0 disk.drbd1965.out0 disk.drbd2078.in0 cpu14.036595855813264" disk.nvme0n1.out 40.5419921875 disk.drbd1931.out0 disk.drbd1828.out0 disk.drbd1226.in0 disk.dm-0.in 2.47265625 disk.dm-0.out 40.5419921875 disk.drbd2111.out0 disk.drbd1550.in0 disk.drbd1550.out0 disk.rbd0.out0 disk.drbd2118.out0* proc.memory.availableMB38517.12109375 disk.drbd1356.in0 disk.rbd1.out 6.267578125 disk.sda1.in0% memory_percent41.227537393569946 disk.drbd1510.out0 disk.nvme0n1.in 2.47265625 disk.drbd1673.out0 disk.rbd0.in0 disk.drbd1285.out0) proc.memory.percent17.18212366104126 disk.drbd1938.out0# proc.memory.rssMB 11260.4765625 disk.drbd1965.in0$ disk./.usageGB578.3443641662598 disk.drbd1931.in0 disk.drbd1584.in0 disk.sda2.out 42.6953125 disk.drbd2111.in0D (: ̜U gpu.0.gpu27.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts120.985' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent40.32833333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors07S (:  ҧU disk.sda2.out 42.9921875 disk.drbd1226.out0 disk.drbd2111.in0 disk.drbd1552.out0 disk.drbd1226.in0 disk.drbd1817.out0 disk.drbd1938.in0* disk./.usagePercent32.343910686061726 disk.drbd1931.out0 disk.drbd1285.in0 proc.cpu.threads29 disk.drbd2078.in0 disk.rbd1.out 6.361328125 disk.sda.in0# proc.memory.rssMB 11260.4765625 cpu14.031645911958583 disk.drbd1817.in0 disk.drbd2041.in0 disk.rbd0.in0 network.sent 3.243529e+06 disk.drbd2118.in0 disk.drbd1965.out0 disk.drbd1673.out0 disk.drbd1828.in0 disk.drbd1965.in0 disk.drbd2111.out0 disk.drbd2041.out0 disk.drbd1550.out0 disk.dm-0.out 40.7763671875 disk.drbd2078.out0 disk.drbd1673.in0 disk.drbd1510.out0 disk.sda2.in0 disk.drbd1552.in0 disk.drbd1828.out0 disk.drbd1289.out0 disk.sda1.out0$ disk./.usageGB578.3419723510742 disk.drbd1931.in0 disk.drbd1289.in0 disk.drbd1285.out0* proc.memory.availableMB38518.03515625 network.recv248673 disk.dm-0.in 2.47265625 disk.drbd1584.out0 disk.drbd1550.in0 disk.drbd1938.out0 disk.rbd0.out0) proc.memory.percent17.18212366104126 disk.sda.out 42.9921875 disk.sda1.in0 disk.drbd1510.in0 disk.drbd1584.in0# memory_percent41.2261426448822 disk.drbd2118.out0" disk.nvme0n1.out 40.7763671875 disk.rbd1.in0 disk.nvme0n1.in 2.47265625 disk.drbd1356.in0 disk.drbd1356.out00ȳ2 - _wandbruntime1564  _runtime15640  _runtime1564.900641983 _step276 & lr-AdamW/pg19.017688051208604e-05 & lr-AdamW/pg29.017688051208604e-05 trainer/global_step3499 % _timestamp1.7857968520776412e+09M6:  & lr-AdamW/pg19.017688051208604e-05 & lr-AdamW/pg29.017688051208604e-05 trainer/global_step3499 % _timestamp1.7857968520776412e+09 _runtime1564.900641983 _step276<2 - _wandbruntime1564  _runtime1564  O XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated13781897 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated10024597 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated23255079 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3079413261312 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated60221524 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed63167522000384 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3079325831680 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated60088213208064 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed36956789 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed23806247 % _timestamp1.7857968520786326e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed60211866 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current245 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed61666937119452 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed23255077 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3162684261888 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3079325831680 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed36956789 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed63167522000384 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated36966445 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current247 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated58595164836832 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated63167626469376 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 3079413261312 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed60230733134848 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated60230758038528 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 ]JDeviceStatsMonitor.on_train_baRe tch_start/inactive_split_bytes.all.allocated63393572974592 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed23255077 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated23806494 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3162814936064 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 3071875125104 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 _step277 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current2518' O E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated36966445 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed63393417396736 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed10024352 _runtime1564.901889558 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated60088213208064 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed13781895 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated61667039961936 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed58595147797472 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 trainer/global_step3499 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3079325831680 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed36956789 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed63167522000384 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated36966445 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current247 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated58595164836832 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated63167626469376 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 3079413261312 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed60230733134848 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated60230758038528 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated63393572974592 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed23255077 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated23806494 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3162814936064 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 3071875125104 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 _step277 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated13781897 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated10024597 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated23255079 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3079413261312 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated60221524 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed63167522000384 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3079325831680 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated60088213208064 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed36956789 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed23806247 % _timestamp1.7857968520786326e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed60211866 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current245 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed61666937119452 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed23255077 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3162684261888 IDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed10027116 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3163583279104 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed23261720 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 _step278 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated60238704 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current248 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed60247940251648 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3163722334720 trainer/global_step3499 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed60229032 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed23261720 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated10027364 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current250 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed36967312 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated60238704 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 % _timestamp1.7857968524634862e+09 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed58611887969792 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 163959296 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated60247965155328 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 3080292888064 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3080292888064 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3080205451264 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 3080205451264 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed23812948 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated36976982 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 _step278 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated60238704 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current248 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed60247940251648 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3163722334720 trainer/global_step3499 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed60229032 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed23261720 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated10027364 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current250 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed36967312 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated60238704 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 % _timestamp1.7857968524634862e+09 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed58611887969792 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 163959296 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated60247965155328 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 3080292888064 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3080292888064 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3080205451264 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 3080205451264 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed23812948 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated36976982 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed10027116 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3163583279104 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed23261720 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144٭D2 - _wandbruntime1565  _runtime1565o&  0loss/SRqqt1/detection_lossi1.565968515118584 .loss/FBt2/detection_loss6.893842267990112 /loss/SRqqt2/assignment_loss1.6600979317228 1loss/FBt1/assignment_loss0.07033875917891394 &loss/total_loss28.521775074594082 trainer/global_step3499 _runtime1565.288435913 1loss/FBt2/assignment_loss0.42909029088914397 -loss/FRt2/detection_loss2.54951443385921 /loss/FBt1/detection_loss2.7976807898041964 epoch0 % _timestamp1.7857968524655538e+09 2loss/SRqqt1/assignment_loss1.6340167291702763 0loss/SRqqt2/detection_loss1.290287023956727 _step279 0loss/FRt1/assignment_loss3.9458122317855424 /loss/FRt2/assignment_loss3.402000238236628 .loss/FRt1/detection_loss2.283125862881944f}'  /loss/SRqqt2/assignment_loss1.6600979317228 1loss/FBt1/assignment_loss0.07033875917891394 &loss/total_loss28.521775074594082 trainer/global_step3499 _runtime1565.288435913 1loss/FBt2/assignment_loss0.42909029088914397 -loss/FRt2/detection_loss2.54951443385921 /loss/FBt1/detection_loss2.7976807898041964 epoch0 % _timestamp1.7857968524655538e+09 2loss/SRqqt1/assignment_loss1.6340167291702763 0loss/SRqqt2/detection_loss1.290287023956727 _step279 0loss/FRt1/assignment_loss3.9458122317855424 /loss/FRt2/assignment_loss3.402000238236628 .loss/FRt1/detection_loss2.283125862881944 0loss/SRqqt1/detection_loss1.565968515118584 .loss/FBt2/detection_loss6.893842267990112*6S (: ܝU gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts98.062' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent32.687333333333335 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0$~N (:  U disk.drbd2118.in0 disk.drbd1356.out0 disk.drbd2078.out0 cpu14.027165840142692 proc.cpu.threads29 disk.drbd1931.out0 disk.drbd2041.in0 disk.drbd1550.out0 disk.sda.out43.34375 disk.sda1.in0 disk.drbd1584.in0 disk.sda2.in0 disk.drbd2111.in0 disk.drbd1828.in0) proc.memory.availableMB 38517.9609375 disk.drbd1931.in0 disk.drbd1938.out0 disk.drbd1584.out0% memory_percent41.226255893707275 disk.sda1.out0 disk.drbd1356.in0 disk.drbd1552.in0 disk.drbd1289.out0 disk.rbd0.in0 disk.drbd2111.out0 disk.drbd1965.out0 disk.drbd1938.in0 disk.drbd1510.out0 disk.drbd2118.out0" disk.nvme0n1.out 41.2294921875 disk.sda2.out43.34375 disk.drbd2041.out0 disk.drbd1828.out0 disk.drbd1285.in0 disk.drbd1510.in0 disk.dm-0.out 41.2294921875 disk.nvme0n1.in 2.47265625 disk.drbd1673.out0 disk.drbd1289.in0# proc.memory.rssMB 11260.4765625) proc.memory.percent17.18212366104126 network.sent 3.28857e+06 disk.drbd1552.out0 disk.drbd1817.in0 disk.rbd1.in0 disk.sda.in0 disk.drbd1226.out0 disk.drbd1817.out0 disk.rbd1.out 6.423828125 disk.drbd1673.in0 disk.drbd2078.in0 network.recv251248) disk./.usagePercent32.34391004604821 disk.drbd1550.in0 disk.dm-0.in 2.47265625$ disk./.usageGB578.3419609069824 disk.drbd1965.in0 disk.rbd0.out0 disk.drbd1285.out0 disk.drbd1226.in02 - _wandbruntime1587  _runtime1587  & lr-AdamW/pg19.146548983635135e-05 & lr-AdamW/pg29.146548983635135e-05 trainer/global_step3549 % _timestamp1.7857968743754635e+09 _runtime1587.198430242 _step2808=  & lr-AdamW/pg19.146548983635135e-05 & lr-AdamW/pg29.146548983635135e-05 trainer/global_step3549 % _timestamp1.7857968743754635e+09 _runtime1587.198430242 _step280w^y2 - _wandbruntime1587  _runtime1587ah' O PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed37484791 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated37494447 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed60946534319104 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3207950965248 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed61091088974848 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current255 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated59432173452832 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated60946551358464 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed13978745 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3115677785488 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed62547834198960 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed59432156413472 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 trainer/global_step3549 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated62547937041444 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed61072018 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed64069856286208 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current257 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 3115763588612 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 3123409396736 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated61091113878528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3207816096768 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _runtime1587.199711102 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated64069960755200 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed37484791 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed23587227 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3123321967104 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated37494447 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed60946534319104 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed64069856286208 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated13978747 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 % _timestamp1.7857968743764205e+09 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed64069856286208 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated13978747 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 % _timestamp1.7857968743764205e+09 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed60946534319104 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3207950965248 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed61091088974848 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current255 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated59432173452832 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated60946551358464 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed13978745 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3115677785488 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed62547834198960 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed59432156413472 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 trainer/global_step3549 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated62547937041444 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed61072018 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed64069856286208 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current257 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 3115763588612 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 3123409396736 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated61091113878528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3207816096768 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 28718748882 - _wandbruntime1587  _runtime1587b>  M EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 166056448 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated64088007144960 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed60963701082112 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 3124201586688 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current263 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3208855226368 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed62565451844502 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3208714073600 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 _runtime1587.568669222 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed37495314 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed60963701082112 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated10170721 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated64317176221696 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated61098856 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current265 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated59448913625152 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated61098856 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated60963718121472 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated64088007144960 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed23593870 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated37504984 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed10170458 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated23593872 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 trainer/global_step3549 K>DeviceStatsMonitor.onQ霯_train_batch_end/inactive_split.all.freed24153140 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated13982684 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3124289023488 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed13982682 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current263 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3208855226368 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed62565451844502 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3208714073600 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 _runtime1587.568669222 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed37495314 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed60963701082112 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated10170721 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated64317176221696 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated61098856 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current265 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated59448913625152 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated61098856 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated60963718121472 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated64088007144960 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed23593870 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated37504984 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed10170458 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated23593872 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 trainer/global_step3549 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed24153140 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated13982684 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3124289023488 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed13982682 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 166056448 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated64088007144960 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0k_p2 - _wandbruntime1587  _runtime1587  1loss/SRqqt1/detection_loss1.1712320453667602 &loss/total_loss49.857186206917476 0loss/FRt2/assignment_loss3.6387841737092432 /loss/FRt2/detection_loss2.7494768146703494 epoch0 _runtime1587.569404098 1loss/FBt1/assignment_loss0.08961302604742134 /loss/FRt1/detection_loss2.5020604151852277 trainer/global_step3549 % _timestamp1.7857968747465081e+09 _step283 2loss/SRqqt1/assignment_loss1.6142991416413208 2loss/FBt2/assignment_loss0.055419958300869133 1loss/SRqqt2/detection_loss1.4718980394097494 .loss/FBt1/detection_loss2.513096524447857 /loss/FBt2/detection_loss28.416272972894756 /loss/FRt1/assignment_loss3.909334829875401 2loss/SRqqt2/assignment_loss1.7256982653685238'h   % _timestamp1.7857968747465081e+09 _step283 2loss/SRqqt1/assignment_loss1.6142991416413208 2loss/FBt2/assignment_loss0.055419958300869133 1loss/SRqqt2/detection_loss1.4718980394097494 .loss/FBt1/detection_loss2.513096524447857 /loss/FBt2/detection_loss28.416272972894756 /loss/FRt1/assignment_loss3.909334829875401 2loss/SRqqt2/assignment_loss1.7256982653685238 1loss/SRqqt1/detection_loss1.1712320453667602 &loss/total_loss49.857186206917476 0loss/FRt2/assignment_loss3.6387841737092432 /loss/FRt2/detection_loss2.7494768146703494 epoch0 _runtime1587.569404098 1loss/FBt1/assignment_loss0.08961302604742134 /loss/FRt1/detection_loss2.5020604151852277 trainer/global_step3549§ (: U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts124.119' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent41.373 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0HQ (:  ֹU disk.drbd1289.out0! disk.nvme0n1.out 41.498046875 disk.drbd2111.out0 disk.drbd1289.in0 disk.sda1.out0 disk.sda1.in0$ disk./.usageGB578.3420028686523 disk.drbd1285.in0 disk.drbd1510.out0 disk.dm-0.in 2.47265625 disk.drbd1356.out0 disk.drbd1817.in0 disk.drbd1285.out0 disk.nvme0n1.in 2.47265625 disk.drbd1673.out0 cpu14.022922741249413 disk.drbd1510.in0 disk.drbd1931.out0 disk.drbd2041.in0 disk.drbd1226.in0 disk.sda.in0 disk.sda2.out 43.94921875 proc.cpu.threads29 network.sent 3.288636e+06 disk.drbd1550.in0 disk.drbd1938.in0 disk.drbd1356.in0) proc.memory.percent17.18212366104126 disk.drbd1584.out0 disk.drbd1673.in0 disk.drbd2041.out0 disk.rbd0.out0 disk.drbd1584.in0 disk.drbd2118.in0 disk.dm-0.out 41.498046875 disk.drbd1828.in0 disk.sda.out 43.94921875 disk.drbd2111.in0 disk.rbd1.out 6.423828125 disk.drbd1938.out0 disk.sda2.in0 disk.drbd1931.in0 disk.drbd1552.in0 disk.drbd1552.out0) disk./.usagePercent32.34391239276446 disk.drbd1550.out0 disk.drbd2078.in0 disk.drbd2118.out0# proc.memory.rssMB 11260.4765625 disk.drbd1226.out0 disk.rbd1.in0 disk.drbd1965.out0( proc.memory.availableMB 38517.671875 disk.drbd2078.out0 network.recv251314 disk.rbd0.in0 disk.drbd1817.out0 disk.drbd1965.in0$ memory_percent41.22669696807861 disk.drbd1828.out0_ʊ (: U gpu.0.gpu21.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts124.672' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent41.55733333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0W (:  U disk.drbd1828.out0 disk.drbd1285.in0 disk.dm-0.in 2.47265625 disk.drbd1938.in0 disk.drbd1938.out0 disk.sda1.out0 disk.drbd1285.out0 disk.drbd1510.out0 disk.drbd1584.in0 disk.drbd1673.out0 disk.drbd1931.out0 disk.drbd1226.in0 disk.drbd1226.out0# disk.nvme0n1.out42.56103515625 network.sent 3.336323e+06 disk.drbd2111.in0 disk.drbd2111.out0 disk.drbd2041.out0 disk.nvme0n1.in 2.47265625 disk.drbd1510.in0 disk.dm-0.out42.56103515625 disk.drbd1356.in0 disk.drbd1552.in0 disk.drbd1289.out0 disk.drbd2078.out0 disk.drbd1965.out0 disk.rbd0.in0 disk.drbd1584.out0 cpu14.018677912459173$ disk./.usageGB578.3421287536621 disk.drbd1931.in0* proc.memory.availableMB38517.92578125 network.recv253875 disk.rbd1.out 6.4560546875 disk.drbd1965.in0 disk.drbd1673.in0 disk.drbd2041.in0 disk.drbd1552.out0 disk.sda1.in0 disk.sda2.in0$ memory_percent41.22630953788757 disk.rbd1.in0 disk.rbd0.out0 disk.drbd2118.in0 disk.drbd2118.out0) proc.memory.percent17.18212366104126 disk.drbd1289.in0 disk.drbd1817.in0 disk.drbd1828.in0 disk.drbd1817.out0 disk.drbd1550.out0 disk.drbd1356.out0* disk./.usagePercent32.343919432913204 disk.drbd1550.in0# proc.memory.rssMB 11260.4765625 proc.cpu.threads29 disk.sda.in0 disk.sda.out 44.1796875 disk.drbd2078.in0 disk.sda2.out 44.1796875(Z2 - _wandbruntime1609  _runtime1609I  & lr-AdamW/pg19.275409916061665e-05 & lr-AdamW/pg29.275409916061665e-05 trainer/global_step3599 % _timestamp1.7857968963547482e+09 _runtime1609.177817437 _step284"  % _timestamp1.7857968963547482e+09 _runtime1609.177817437 _step284 & lr-AdamW/pg19.275409916061665e-05 & lr-AdamW/pg29.275409916061665e-05 trainer/global_step3599A2 - _wandbruntime1609  _runtime1609F4M' O PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed38012793 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed60269165029472 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current243 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated61951469718528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 _runtime1609.179278366 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated23919379 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated61804889508864 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3167405532160 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated24487137 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated65204591089152 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated23919379 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed61804872469504 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated60269182068832 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed65204433414144 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed23919377 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed10311299 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed38012793 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3167318102528 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated61941828 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated14175597 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 $ _timestamp1.785796896355916e+09 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3252988599296 trainer/global_step3599 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated63428834120952 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 3159652052120 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated38022449 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed24486894 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated61804889508864 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 _step285 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed23919377 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed61951444814848 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed63428731278468 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed64972190572032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed61932170 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated64972295041024 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed61804872469504 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed14175595 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current241 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3253121370624 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated64972295041024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated10311540 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3159566248996 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0Xt O WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3252988599296 trainer/global_step3599 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated63428834120952 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 3159652052120 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated38022449 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed24486894 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated61804889508864 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 _step285 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed23919377 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed61951444814848 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed63428731278468 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed64972190572032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed61932170 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated64972295041024 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed61804872469504 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed14175595 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current241 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3253121370624 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 UBDeviceS@H*NtatsMonitor.on_train_batch_start/active_bytes.all.allocated64972295041024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated10311540 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3159566248996 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed38012793 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed60269165029472 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current243 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated61951469718528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 _runtime1609.179278366 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated23919379 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated61804889508864 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3167405532160 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated24487137 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated65204591089152 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated23919379 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed61804872469504 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated60269182068832 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed65204433414144 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed23919377 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed10311299 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed38012793 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3167318102528 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated61941828 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated14175597 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 $ _timestamp1.785796896355916e+09d|2 - _wandbruntime1609  _runtime1609Ɍ~& M VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated10314429 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated61959008 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3253904115200 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed38023316 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed10314186 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated23926022 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated61959008 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated61822056271872 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed63446348924010 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3160529525454 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed38023316 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed61822039232512 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed24493718 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed60285905201792 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed14179532 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 _runtime1609.564408962 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed64990236954624 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated61968676835328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed23926020 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current245 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated64990341430784 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed61949336 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current243 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated65222715811840 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed61968651931648 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 % _timestamp1.7857968967409651e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3168285158912 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated24493963 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3254038976512 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 3160443722218 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 trainer/global_step3599 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3168197722112 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3168285158912 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated24493963 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3254038976512 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 3160443722218 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 trainer/global_step3599 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3168197722112 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated61959008 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3253904115200 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed38023316 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed10314186 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated23926022 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated61959008 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated61822056271872 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed63446348924010 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3160529525454 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed38023316 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed61822039232512 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed24493718 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed60285905201792 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed14179532 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 _runtime1609.564408962 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed64990236954624 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated61968676835328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251%2 - _wandbruntime1609  _runtime1609   /loss/FRt1/assignment_loss4.089683898771652 1loss/FBt1/assignment_loss0.11796619942348723 1loss/FBt2/assignment_loss0.47210086311679333 0loss/SRqqt1/detection_loss1.647885705901058 _step287 /loss/FRt1/detection_loss2.5232748112293204 1loss/SRqqt2/detection_loss1.3822780710210532 /loss/FBt1/detection_loss2.2686906493441197 _runtime1609.565600159 0loss/FRt2/assignment_loss3.1567550530800452 2loss/SRqqt1/assignment_loss1.7149794744466669 2loss/SRqqt2/assignment_loss1.3191866814855515 /loss/FRt2/detection_loss3.0805454342984238 .loss/FBt2/detection_loss4.608960136771202 &loss/total_loss26.382306978889375 epoch0 trainer/global_step3599 $ _timestamp1.785796896742769e+09bf   1loss/SRqqt2/detection_loss1.3822780710210532 /loss/FBt1/detection_loss2.2686906493441197 _runtime1609.565600159 0loss/FRt2/assignment_loss3.1567550530800452 2loss/SRqqt1/assignment_loss1.7149794744466669 2loss/SRqqt2/assignment_loss1.3191866814855515 /loss/FRt2/detection_loss3.0805454342984238 .loss/FBt2/detection_loss4.608960136771202 &loss/total_loss26.382306978889375 epoch0 trainer/global_step3599 $ _timestamp1.785796896742769e+09 /loss/FRt1/assignment_loss4.089683898771652 1loss/FBt1/assignment_loss0.11796619942348723 1loss/FBt2/assignment_loss0.47210086311679333 0loss/SRqqt1/detection_loss1.647885705901058 _step287 /loss/FRt1/detection_loss2.5232748112293204..ř (: ӍU gpu.0.gpu0.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts94.36' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.453333333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0XiƅM (:  U disk.sda2.out 44.515625 disk.sda2.in0 disk.drbd2041.in0 disk.rbd0.out0) proc.memory.percent17.18212366104126 disk.drbd2118.in0 disk.drbd1965.out0 disk.sda.out 44.515625 disk.drbd1356.in0 disk.drbd1931.out0 proc.cpu.threads29) disk./.usagePercent32.34391665952127! disk.nvme0n1.out 42.916015625 disk.drbd1931.in0 disk.drbd1552.in0 disk.sda.in0 disk.drbd1285.out0# proc.memory.rssMB 11260.4765625 disk.drbd1965.in0( proc.memory.availableMB 38517.828125 disk.drbd1226.in0 disk.nvme0n1.in 2.47265625 disk.drbd1356.out0 disk.drbd1817.in0 disk.drbd1938.out0$ memory_percent41.22645854949951 network.sent 3.381446e+06 disk.drbd1584.out0 disk.sda1.in0 disk.drbd1510.in0 disk.drbd1510.out0$ disk./.usageGB578.3420791625977 disk.drbd1817.out0 disk.drbd1550.in0 disk.drbd1289.out0 disk.drbd1938.in0 disk.dm-0.out 42.916015625 disk.drbd1584.in0 disk.drbd1673.out0 disk.drbd2041.out0 disk.rbd0.in0 disk.drbd1285.in0 cpu14.01451015268048 network.recv256229 disk.drbd2078.in0 disk.drbd2111.out0 disk.rbd1.in0 disk.sda1.out0 disk.drbd1226.out0 disk.drbd2111.in0 disk.drbd1673.in0 disk.drbd1552.out0 disk.drbd1828.in0 disk.drbd1289.in0 disk.dm-0.in 2.47265625 disk.drbd2078.out0 disk.drbd2118.out0 disk.drbd1828.out0 disk.drbd1550.out0 disk.rbd1.out 6.6357421875#2 - _wandbruntime1631  _runtime1631  _runtime 1631.20034614 _step288 & lr-AdamW/pg19.404270848488195e-05 & lr-AdamW/pg29.404270848488195e-05 trainer/global_step3649 % _timestamp1.7857969183773875e+09XG  trainer/global_step3649 % _timestamp1.7857969183773875e+09 _runtime 1631.20034614 _step288 & lr-AdamW/pg19.404270848488195e-05 & lr-AdamW/pg29.404270848488195e-05;2 - _wandbruntime1631  _runtime1631Ro  O _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated62663227659264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated65874629326848 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current259 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated66110115175424 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed65874524857856 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed62663210619904 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated65874629326848 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed62811800654848 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3203454712504 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 3211401667584 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed66109953306112 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated64309731200460 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed24251527 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed61106173645472 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed65874524857856 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3211314237952 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated24251529 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated61106190684832 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 _runtime1631.201614643 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed62663210619904 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed64309628357976 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated24251529 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 trainer/global_step3649 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated38550451 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated62663227659264 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated38550451 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated62811825558528 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed65874524857856 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed62663210619904 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated65874629326848 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed62811800654848 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3203454712504 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 3211401667584 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed66109953306112 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated64309731200460 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed24251527 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed61106173645472 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed65874524857856 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3211314237952 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated24251529 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated61106190684832 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 _runtime1631.201614643 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed62663210619904 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed64309628357976 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated24251529 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 trainer/global_step3649 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632O%2 - _wandbruntime1631  _runtime1631}?& M E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 _step290 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated24258172 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed64327246003518 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated62829032675328 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current271 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 3204332185726 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed62809488 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated24258172 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated10457281 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed66128058997760 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 $ _timestamp1.785796918748668e+09 trainer/global_step3649 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed38551318 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated65892675716608 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 3212193857536 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated62680394422272 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3212193857536 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed24833394 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed62809488 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated61122930857152 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3299188184576 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated38560988 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated62819160 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed14376382 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed61122913817792 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed10457012 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated62819160 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated62680394422272 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated66128220859904 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3299051226112 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 3212281294336 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3212281294336 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 _runtime1631.572106475 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed62680377382912 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed38551318 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated24833665 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0?T M NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated38560988 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated62819160 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed14376382 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed61122913817792 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed10457012 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated62819160 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated62680394422272 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated66128220859904 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3299051226112 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 3212281294336 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3212281294336 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 _runtime1631.572106475 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed62680377382912 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed38551318 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated24833665 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 _step290 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated24258172 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed64327246003518 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated62829032675328 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current271 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 3204332185726 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed62809488 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated24258172 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated10457281 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed66128058997760 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 SDDeviceStatsMonitor.on_train_batch_end/resAYAerved_bytes.large_pool.peak 2768240640 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 $ _timestamp1.785796918748668e+09 trainer/global_step3649 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed38551318 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated65892675716608 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 3212193857536 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated62680394422272 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3212193857536 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed24833394 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed62809488 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated61122930857152 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3299188184576 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections09i2 - _wandbruntime1631  _runtime1631}\6  /loss/FBt2/assignment_loss0.503460667071421 1loss/SRqqt2/detection_loss1.4644841919080533 &loss/total_loss24.968172933195355 2loss/FBt1/assignment_loss0.034689741060344295 /loss/FRt1/detection_loss2.4495941985000687 -loss/FBt1/detection_loss2.02627039475155 epoch0 trainer/global_step3649 % _timestamp1.7857969187505324e+09 0loss/FRt1/assignment_loss3.9185380495867683 1loss/SRqqt2/assignment_loss1.540925683843535 .loss/FRt2/detection_loss2.969254765724927 1loss/SRqqt1/detection_loss1.6509787422984363 /loss/FBt2/detection_loss3.1909081273608737 _runtime1631.573374664 .loss/FRt2/assignment_loss3.48810539120122 2loss/SRqqt1/assignment_loss1.7309629798881592 _step291  2loss/FBt1/assignment_loss0.034689741060344295 /loss/FRt1/detection_loss2.4495941985000687 -loss/FBt1/detection_loss2.02627039475155 epoch0 trainer/global_step3649 % _timestamp1.7857969187505324e+09 0loss/FRt1/assignment_loss3.9185380495867683 1loss/SRqqt2/assignment_loss1.540925683843535 .loss/FRt2/detection_loss2.969254765724927 1loss/SRqqt1/detection_loss1.6509787422984363 /loss/FBt2/detection_loss3.1909081273608737 _runtime1631.573374664 .loss/FRt2/assignment_loss3.48810539120122 2loss/SRqqt1/assignment_loss1.7309629798881592 _step291 /loss/FBt2/assignment_loss0.503460667071421 1loss/SRqqt2/detection_loss1.4644841919080533 &loss/total_loss24.968172933195355nCkV (:  U) disk./.usagePercent32.34392092627809 disk.drbd1285.out0# proc.memory.rssMB 11260.4765625 disk.sda2.in0 disk.drbd1938.out0 disk.drbd2078.out0 disk.sda1.out0 cpu14.010557340129779 network.sent 3.384587e+06 disk.drbd1931.in0 disk.sda.in0 disk.drbd1510.in0 disk.drbd1226.out0 disk.rbd1.out 6.6982421875 disk.drbd1931.out0 disk.drbd2041.out0 disk.dm-0.in 2.47265625# disk./.usageGB578.342155456543 disk.drbd1828.in0( proc.memory.availableMB 38518.515625) proc.memory.percent17.18212366104126 disk.nvme0n1.in 2.47265625 disk.drbd1965.out0 disk.drbd1673.in0 disk.drbd1356.in0 disk.drbd1356.out0 disk.drbd1965.in0 disk.drbd2041.in0 disk.drbd2118.in0 disk.sda2.out 44.91796875 disk.rbd1.in0 disk.drbd1289.out0 disk.drbd1285.in0% memory_percent41.225409507751465# disk.nvme0n1.out44.03173828125 disk.drbd1817.out0 disk.drbd1673.out0 disk.drbd1552.in0 disk.drbd1550.out0 disk.sda1.in0 disk.drbd1510.out0 disk.drbd2111.in0 disk.drbd1552.out0 disk.drbd1584.in0 disk.drbd2118.out0 proc.cpu.threads29 disk.drbd1817.in0 disk.drbd1289.in0 disk.sda.out 44.91796875 disk.drbd1226.in0 disk.rbd0.out0 network.recv257198 disk.drbd1828.out0 disk.drbd2078.in0 disk.drbd2111.out0 disk.drbd1550.in0 disk.drbd1938.in0 disk.rbd0.in0 disk.dm-0.out44.03173828125 disk.drbd1584.out0vZ (: V gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts108.995' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent36.33166666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0ؤ (: ɲU gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts121.968' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent40.656000000000006 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0-ՍS (:  ɲU disk.nvme0n1.in 2.47265625 disk.rbd1.in0 disk.drbd1965.in0 disk.drbd2041.out0 disk.drbd1550.in0$ memory_percent41.22660160064697 disk.rbd1.out 6.6982421875 disk.drbd1226.out0 disk.drbd2078.in0 disk.drbd1673.out0 network.recv259713 disk.drbd1510.out0 disk.sda2.out 45.2734375 disk.drbd1552.in0 disk.drbd1289.out0 disk.drbd1285.out0" disk.nvme0n1.out 44.4130859375 disk.drbd1931.out0 disk.sda.out 45.2734375* disk./.usagePercent32.344185465200745 disk.drbd2111.out0 disk.rbd0.in0 disk.drbd1356.in0 disk.drbd1550.out0 disk.drbd1817.out0 disk.drbd1673.in0 disk.drbd1817.in0 disk.drbd2041.in0 disk.drbd1289.in0 disk.dm-0.out 44.4130859375 disk.drbd2118.out0 disk.drbd2111.in0 disk.drbd1965.out0 disk.rbd0.out0$ disk./.usageGB578.3468856811523 disk.drbd1226.in0 disk.drbd1584.out0 disk.drbd2118.in0 disk.drbd1938.in0 proc.cpu.threads29 disk.sda1.in0 disk.drbd1510.in0 disk.dm-0.in 2.47265625 disk.drbd1828.out0 disk.drbd1938.out0 cpu14.006242299672461 disk.drbd2078.out0 disk.drbd1584.in0 disk.drbd1552.out0( proc.memory.availableMB 38517.734375) proc.memory.percent17.18212366104126 network.sent 3.429678e+06 disk.sda2.in0 disk.sda1.out0 disk.drbd1356.out0 disk.drbd1828.in0 disk.drbd1285.in0 disk.drbd1931.in0 disk.sda.in0# proc.memory.rssMB 11260.47656252 - _wandbruntime1653  _runtime1653QE  $ _timestamp1.785796940462367e+09 _runtime1653.286719414 _step292 & lr-AdamW/pg19.533131780914726e-05 & lr-AdamW/pg29.533131780914726e-05 trainer/global_step3699t:  $ _timestamp1.785796940462367e+09 _runtime1653.286719414 _step292 & lr-AdamW/pg19.533131780914726e-05 & lr-AdamW/pg29.533131780914726e-05 trainer/global_step3699m)2 - _wandbruntime1653  _runtime1653Jy+' O TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated24583679 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed14569295 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed63672156494848 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3343381678592 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed66776859143680 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3255310373376 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed66776859143680 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed39068797 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed65190525437484 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3247343176012 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 % _timestamp1.7857969404631307e+09 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated63521565809664 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3255397803008 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current254 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 trainer/global_step3699 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated39078453 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed10598009 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 3247428979136 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed63521548770304 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 3255397803008 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated66776963612672 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3255310373376 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed67015538173440 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 _step293 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated14569297 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated25167558 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated67015697945600 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated61943199300832 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated24583679 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3343516547072 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 % _timestamp1.7857969404631307e+09 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated63521565809664 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3255397803008 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current254 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 trainer/global_step3699 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated39078453 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed10598009 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 3247428979136 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed63521548770304 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 3255397803008 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated66776963612672 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3255310373376 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed67015538173440 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 _step293 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated14569297 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated25167558 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated67015697945600 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated61943199300832 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated24583679 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3343516547072 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3255310373376 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed66776859143680 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed39068797 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed65190525437484 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3247343176012 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed25174072 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated25174302 _step294 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated$8 3344418031104 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed61959922433792 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed39079320 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed39079320 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3256189992960 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated24590322 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated66795010002432 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated63689388515328 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated63538732572672 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 trainer/global_step3699 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed63538715533312 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated65208245925622 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated67033806546432 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed65208143083026 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated63679312 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated63679312 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current228 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 3256189992960 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated24590322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed63538715533312 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated10601068 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3256277429760 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated39088990 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3248306452470 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed63669640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 3256277429760 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed24590320 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 % _timestamp1.7857969408342228e+09 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed63669640 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 3248220649234 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _runtime1653.657578187 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current230 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed67033650975744 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed63689363611648 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed10600840 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3344287364096 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed39079320 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3256189992960 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated24590322 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated66795010002432 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated63689388515328 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated63538732572672 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 trainer/global_step3699 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed63538715533312 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated65208245925622 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated67033806546432 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed65208143083026 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated63679312 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated63679312 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current228 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 3256189992960 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated24590322 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed63538715533312 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated10601068 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3256277429760 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated39088990 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3248306452470 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed63669640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 3256277429760 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed24590320 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 % _timestamp1.7857969408342228e+09 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed63669640 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 3248220649234 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _runtime1653.657578187 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current230 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed67033650975744 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed63689363611648 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed10600840 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3344287364096 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed25174072 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated25174302 _step294 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3344418031104 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed61959922433792 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed39079320 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 EDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed62780190877472 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed25507313 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated39606455 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 _runtime1675.585532233 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3299306508800 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated67921260547072 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed67921098677760 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3388723308544 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed10741168 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated64379903960064 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated64522284 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated62780207916832 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed39596799 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3291231639520 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed67679193429504 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3299393938432 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated66071525359476 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed67679193429504 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed64379886920704 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed14766145 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed24915827 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed24915827 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current254 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3299306508800 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed66071422516992 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251< O [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed64379886920704 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current256 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed67679193429504 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3299393938432 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated66071525359476 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed67679193429504 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed64379886920704 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed14766145 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed24915827 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed24915827 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current254 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3299306508800 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed66071422516992 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 trainer/global_step3749 % _timestamp1.7857969627622697e+09 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated14766147 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed62780190877472 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed25507313 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated39606455 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 _runtime1675.585532233 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3299306508800 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated67921260547072 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed67921098677760 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3388723308544 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed10741168 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated64379903960064 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated64522284 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated62780207916832 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed39596799 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3291231639520 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 IDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated62796948089152 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed24922470 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 trainer/global_step3749 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated25514211 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed66089040162534 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated64539464 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed64529792 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed39607322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated24922472 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed64549719451648 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed39607322 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed25513973 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated64549744355328 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated24922472 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed64397053683712 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current236 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated39616992 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed14770082 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3389624655360 % _timestamp1.7857969631379423e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed67939211342848 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 3292109112742 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed64397053683712 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3389491891200 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated67697344288256 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated66089143005130 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3292194915978 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed67697239812096 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed64529792 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current238Cpu& M J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed24922470 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3300273565184 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated64539464 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed64529792 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed39607322 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated24922472 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed64549719451648 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed39607322 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed25513973 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated64549744355328 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated24922472 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed64397053683712 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current236 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated39616992 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed14770082 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3389624655360 % _timestamp1.7857969631379423e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed67939211342848 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 3292109112742 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed64397053683712 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3389491891200 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated67697344288256 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated66089143005130 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3292194915978 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed67697239812096 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed64529792 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current238 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 _runtime1675.961271978 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated62796948089152 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed24922470 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 trainer/global_step3749 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated25514211 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed66089040162534 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 ODeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated63617216532832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed65238225071104 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3343302644224 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated66952422438984 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed68826670306304 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated25247979 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed25846181 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed68581527715328 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3343302644224 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated68581632184320 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed65238225071104 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated10883434 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current248 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed10883186 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed14962995 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed65392868174848 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated40134457 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3335120103028 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed63617199493472 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated68581632184320 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3343390073856 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 3343390073856 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 % _timestamp1.7857969844031963e+09 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed68581527715328#' O ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 _step301 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated40134457 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed25247977 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated65382436 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated65392893078528 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 3335205906152 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed68581527715328 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3343302644224 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated68581632184320 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed65238225071104 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated10883434 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current248 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed10883186 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed14962995 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed65392868174848 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated40134457 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3335120103028 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed63617199493472 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated68581632184320 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3343390073856 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 3343390073856 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 % _timestamp1.7857969844031963e+09 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed68581527715328 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current250 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated63617216532832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed65238225071104 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3343302644224 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated66952422438984 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed68826670306304 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated25247979 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed25846181 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed65255391834112 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated68599678574080 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated65399616 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated66970040084638 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed65255391834112 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed14966932 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed40135324 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3434708046336 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3344182263808 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed68844783337984 % _timestamp1.7857969847784827e+09 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated40144994 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed25852915 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3434836616192 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 _step302 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 3344182263808 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed25254620 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current232 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed40135324 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed65389944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed25254620 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated25853149 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated14966934 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated65399616 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated25254622 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed63633939665792 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3344269700608 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated68844936811520 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3336083379486 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed10885983 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated65410100195328 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 _runtime1697.601825598 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current234 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated68599678574080 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed66969937242042 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 trainer/global_step3799 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed65255391834112 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated68599678574080 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated65399616 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated66970040084638 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed65255391834112 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed14966932 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed40135324 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3434708046336 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3344182263808 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed68844783337984 % _timestamp1.7857969847784827e+09 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated40144994 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed25852915 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3434836616192 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 _step302 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 3344182263808 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed25254620 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current232 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed40135324 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed65389944 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed25254620 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated25853149 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated14966934 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated65399616 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated25254622 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed63633939665792 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3344269700608 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated68844936811520 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3336083379486 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed10885983 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated65410100195328 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 _runtime1697.601825598 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current234 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated68599678574080 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed66969937242042 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 trainer/global_step3799 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed69732240654336 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated40662459 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated67833319518492 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3387298779648 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed69483862001152 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated69732396232192 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated64454225148832 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated66096580260864 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current245 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3479016639488 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3387298779648 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed25580127 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed26186032 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 _runtime1719.424675801 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed64454208109472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated66242588 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 trainer/global_step3849 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3387386209280 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated66096580260864 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated69483966470144 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed66253224014848 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 3379094369660 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3379008566536 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 % _timestamp1.7857970066012897e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed66232930 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated25580129 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed66096563221504 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated15159847 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538Ko O KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed64454208109472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated66242588 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 trainer/global_step3849 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3387386209280 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated66096580260864 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated69483966470144 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed66253224014848 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 3379094369660 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3379008566536 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 % _timestamp1.7857970066012897e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed66232930 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated25580129 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed66096563221504 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated15159847 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed66096563221504 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated25580129 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed25580127 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 3387386209280 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated26186277 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3479147313664 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed69732240654336 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated40662459 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated67833319518492 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3387298779648 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed69483862001152 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated69732396232192 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated64454225148832 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated66096580260864 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current245 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3479016639488 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3387298779648 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed25580127 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed26186032 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 _runtime1719.424675801 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak118872 - _wandbruntime1719  _runtime1719+IJ& M aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3480055662592 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed25586770 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed26192844 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed69750343544320 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated67850937164146 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 3388178399232 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated69750511697920 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed66270431131648 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated25586772 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed64470948281792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed66113729984512 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed11029062 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed67850834321550 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 143249920 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated40672996 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 % _timestamp1.7857970069667354e+09 _step306 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated66259768 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 3379886039758 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed40663326 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed40663326 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated66259768 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed66250096 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated66113747023872 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated26193106 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated25586772 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current262 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed69501908383744 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed66250096 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated69502012859904 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated11029322 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated64470965321152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 trainer/global_step3849 _runtime1719.790057448 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated15163784 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3479912412672 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3388178399232 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed25586770 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated69502012859904A*& M XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated67850937164146 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 3388178399232 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated69750511697920 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed66270431131648 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated25586772 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed64470948281792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed66113729984512 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed11029062 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed67850834321550 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 143249920 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated40672996 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 % _timestamp1.7857970069667354e+09 _step306 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated66259768 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 3379886039758 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed40663326 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed40663326 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated66259768 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed66250096 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated66113747023872 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated26193106 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated25586772 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current262 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed69501908383744 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed66250096 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated69502012859904 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated11029322 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated64470965321152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 trainer/global_step3849 _runtime1719.790057448 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated15163784 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3479912412672 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3388178399232 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed25586770 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated69502012859904 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3480055662592 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed25586770 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed26192844 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed69750343544320 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320?F{2 - _wandbruntime1719  _runtime1719av  1loss/SRqqt2/assignment_loss1.638992891532101 1loss/FBt1/assignment_loss0.06263120961514232 /loss/FRt1/detection_loss2.2875030048653087 0loss/SRqqt1/detection_loss1.324834001582933 1loss/SRqqt2/detection_loss1.2895489016424755 /loss/FBt1/detection_loss1.3169052809514596 epoch0 trainer/global_step3849 % _timestamp1.7857970069685452e+09 _step307 0loss/FBt2/assignment_loss0.6376479752361774 &loss/total_loss24.642244367241794 _runtime1719.791361011 0loss/FRt2/assignment_loss3.1259192639383775 /loss/FRt2/detection_loss2.7964998201049607 .loss/FBt2/detection_loss4.428425470987956 /loss/FRt1/assignment_loss4.162002981236551 2loss/SRqqt1/assignment_loss1.57133356554835452\  1loss/SRqqt2/assignment_loss1.638992891532101 1loss/FBt1/assignment_loss0.06263120961514232 /loss/FRt1/detection_loss2.2875030048653087 0loss/SRqqt1/detection_loss1.324834001582933 1loss/SRqqt2/detection_loss1.2895489016424755 /loss/FBt1/detection_loss1.3169052809514596 epoch0 trainer/global_step3849 % _timestamp1.7857970069685452e+09 _step307 0loss/FBt2/assignment_loss0.6376479752361774 &loss/total_loss24.642244367241794 _runtime1719.791361011 0loss/FRt2/assignment_loss3.1259192639383775 /loss/FRt2/detection_loss2.7964998201049607 .loss/FBt2/detection_loss4.428425470987956 /loss/FRt1/assignment_loss4.162002981236551 2loss/SRqqt1/assignment_loss1.5713335655483545y3 (: іU gpu.0.gpu0.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts94.504' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.501333333333335 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0aC8N (:  U disk.drbd1550.out0 disk.dm-0.in 2.47265625 disk.drbd1938.in0 disk.drbd1828.out0 disk.drbd1510.out0 disk.rbd1.out 6.9189453125 disk.drbd1673.out0 disk.drbd1673.in0 network.sent 3.569956e+06 network.recv267190 disk.drbd1510.in0 disk.drbd1226.in0 disk.drbd1226.out0 disk.drbd1931.in0' proc.memory.availableMB 38517.84375 disk.rbd0.in0 disk.drbd2118.out0 disk.rbd1.in0 disk.drbd1552.in0 disk.drbd1550.in0# memory_percent41.2264347076416 disk.drbd1289.out0 disk.rbd0.out0 disk.sda1.in0 disk.drbd1584.out0! disk.nvme0n1.out 50.884765625 disk.drbd2111.in0 disk.sda.out 46.9140625 disk.drbd1285.out0 disk.sda2.out 46.9140625 disk.drbd1552.out0# proc.memory.rssMB 11260.4765625) proc.memory.percent17.18212366104126 disk.drbd1931.out0 disk.drbd1828.in0 disk.drbd1289.in0) disk./.usagePercent32.34371974869417 disk.drbd1584.in0 disk.drbd2111.out0$ disk./.usageGB578.3385581970215 disk.drbd1285.in0 disk.dm-0.out 50.884765625 disk.drbd1817.out0 disk.drbd1965.in0 disk.drbd2041.out0 proc.cpu.threads29 cpu13.987283643932098 disk.drbd1356.in0 disk.drbd2041.in0 disk.drbd1938.out0 disk.sda1.out0 disk.drbd1356.out0 disk.sda2.in0 disk.drbd1817.in0 disk.drbd1965.out0 disk.sda.in0 disk.drbd2078.in0 disk.drbd2078.out0 disk.drbd2118.in0 disk.nvme0n1.in 2.47265625eӚ (: ɎU gpu.0.gpu0.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts94.291' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.430333333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors01K (:  U disk.drbd1673.in0 disk.sda1.out0 disk.drbd1356.out0* proc.memory.availableMB38517.02734375 disk.drbd2111.out0 disk.drbd1673.out0 disk.drbd1550.out0 disk.drbd1289.in0 disk.drbd1285.out0$ disk./.usageGB578.3380241394043 disk.drbd1552.in0 disk.sda.in0 disk.drbd1226.in0 disk.drbd1584.in0 disk.nvme0n1.in 2.47265625 disk.nvme0n1.out51.21875 disk.drbd1550.in0 cpu13.98351512534511 disk.drbd1289.out0 disk.drbd1226.out0 network.recv269183 disk.drbd1285.in0$ memory_percent41.22768044471741 disk.rbd1.in0 disk.drbd2041.in0 disk.sda2.in0 network.sent 3.614889e+06 disk.dm-0.out51.21875 disk.drbd1817.in0 disk.drbd1965.in0 disk.drbd1510.out0 disk.drbd1584.out0 proc.cpu.threads29 disk.drbd2111.in0 disk.drbd1828.out0 disk.drbd1938.in0 disk.drbd2118.in0 disk.rbd1.out 7.0986328125 disk.drbd1965.out0 disk.drbd2041.out0 disk.rbd0.out0 disk.drbd1356.in0 disk.drbd1931.in0 disk.drbd1828.in0 disk.sda1.in0 disk.drbd1510.in0 disk.dm-0.in 2.47265625 disk.drbd1817.out0 disk.drbd1938.out0 disk.rbd0.in0 disk.sda.out 47.27734375 disk.drbd2078.in0 disk.sda2.out 47.27734375 disk.drbd1931.out0 disk.drbd1552.out0 disk.drbd2078.out0 disk.drbd2118.out0) proc.memory.percent17.18212366104126) disk./.usagePercent32.34368988139645# proc.memory.rssMB 11260.4765625m22 - _wandbruntime1742  _runtime1742  ' lr-AdamW/pg10.00010048575510620847 ' lr-AdamW/pg20.00010048575510620847 trainer/global_step3899 % _timestamp1.7857970295417218e+09 _runtime1742.364804669 _step308|   ' lr-AdamW/pg20.00010048575510620847 trainer/global_step3899 % _timestamp1.7857970295417218e+09 _runtime1742.364804669 _step308 ' lr-AdamW/pg10.000100485755106208472 - _wandbruntime1742  _runtime1742wp O WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3431294915072 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed41180805 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated70386300755968 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated67113604758528 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 _runtime1742.366440057 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed68714113755516 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated65291233764832 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated25912279 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed25912277 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed70386196286976 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 3422982833168 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed67093082 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3524171330560 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current270 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed67113579854848 _step309 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed66954901371904 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed65291216725472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3431294915072 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 3431382344704 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed41180805 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed15356695 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated70637910957568 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated11169233 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated70386300755968 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed70386196286976 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated66954918411264 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated26525930 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed26525660 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current268 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed70637751185408 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3524306199040 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 % _timestamp1.7857970295430171e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed25912277 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3422897030044 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated67102740 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251yt' O ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 _runtime1742.366440057 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed68714113755516 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated65291233764832 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated25912279 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed25912277 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed70386196286976 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 3422982833168 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed67093082 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3524171330560 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current270 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed67113579854848 _step309 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed66954901371904 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed65291216725472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3431294915072 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 3431382344704 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed41180805 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed15356695 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated70637910957568 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated11169233 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated70386300755968 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed70386196286976 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated66954918411264 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated26525930 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed26525660 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current268 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed70637751185408 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3524306199040 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 % _timestamp1.7857970295430171e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed25912277 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3422897030044 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated67102740 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3431294915072 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed41180805 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated70386300755968 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated67113604758528 IDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated41200998 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated68731834243654 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 3432261971456 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 trainer/global_step3899 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated66972085174272 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current247 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _runtime1742.814362642 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed26532394 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated67119920 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated67130811875328 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated65307973937152 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated70404347145728 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 3432174534656 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 3423774503266 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated25918922 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed66972068134912 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated66972085174272 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed41191328 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed70404242669568 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3432174534656 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated67119920 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated25918922 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed15360632 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3423860306502 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed41191328 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated11172007 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 % _timestamp1.7857970299908884e+09 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3432261971456 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed70655863692800 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current245 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated41200998 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed26532394 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated67119920 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated67130811875328 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated65307973937152 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated70404347145728 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 3432174534656 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 3423774503266 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated25918922 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed66972068134912 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated66972085174272 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed41191328 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed70404242669568 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3432174534656 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated67119920 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated25918922 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed15360632 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3423860306502 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed41191328 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated11172007 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 % _timestamp1.7857970299908884e+09 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3432261971456 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed70655863692800 [MDeviceStatsMonitor.on_train_batch_end/inactive _split_bytes.small_pool.current 130667008 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current245 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated41200998 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated41200998 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated68731834243654 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 3432261971456 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 trainer/global_step3899 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated66972085174272 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current247 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _runtime1742.814362642 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251i\2 - _wandbruntime1742  _runtime1742(9  /loss/FRt1/assignment_loss3.929548153337443 1loss/SRqqt2/detection_loss1.6229064976916956 2loss/SRqqt2/assignment_loss1.8124993798525437 /loss/FRt1/detection_loss2.3545534166524993 trainer/global_step3899 2loss/SRqqt1/assignment_loss1.3303180175312495 1loss/FBt1/assignment_loss0.09843907518894376 0loss/FBt2/assignment_loss0.7904655230619634 /loss/FRt2/detection_loss2.7590117174862088 /loss/FBt1/detection_loss2.5194129629624977 &loss/total_loss24.947225534069936 /loss/FRt2/assignment_loss4.025670065396074 0loss/SRqqt1/detection_loss1.241842428917129 .loss/FBt2/detection_loss2.462558295991686 epoch0 % _timestamp1.7857970299931588e+09 _runtime1742.816041785 _step311?Q  % _timestamp1.7857970299931588e+09 _runtime1742.816041785 _step311 /loss/FRt1/assignment_loss3.929548153337443 1loss/SRqqt2/detection_loss1.6229064976916956 2loss/SRqqt2/assignment_loss1.8124993798525437 /loss/FRt1/detection_loss2.3545534166524993 trainer/global_step3899 2loss/SRqqt1/assignment_loss1.3303180175312495 1loss/FBt1/assignment_loss0.09843907518894376 0loss/FBt2/assignment_loss0.7904655230619634 /loss/FRt2/detection_loss2.7590117174862088 /loss/FBt1/detection_loss2.5194129629624977 &loss/total_loss24.947225534069936 /loss/FRt2/assignment_loss4.025670065396074 0loss/SRqqt1/detection_loss1.241842428917129 .loss/FBt2/detection_loss2.462558295991686 epoch0ڍ} (: 캄U gpu.0.gpu0.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts94.379' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent31.45966666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0U (:  U disk.dm-0.out 51.958984375 network.recv271594 disk.sda1.in0 disk.drbd2041.in0 disk.drbd1828.in0 disk.drbd1510.in0 disk.drbd2111.in0 disk.nvme0n1.in 2.47265625 disk.rbd1.in0 disk.drbd1673.out0 disk.sda.out 47.57421875 disk.drbd1285.in0 disk.drbd2078.out0 disk.drbd2118.out0 disk.drbd2111.out0 disk.drbd1931.in0% memory_percent41.227394342422485 disk.drbd2078.in0 disk.drbd1584.out0 disk.drbd1965.in0 disk.drbd1289.in0) proc.memory.percent17.18212366104126 proc.cpu.threads29 disk.sda1.out0 disk.drbd1965.out0 disk.drbd1817.out0 disk.drbd1552.in0 network.sent 3.659936e+06 disk.drbd1510.out0 disk.drbd1584.in0 disk.drbd1817.in0 disk.rbd0.in0 disk.rbd0.out0* proc.memory.availableMB38517.21484375$ disk./.usageGB578.3409957885742 disk.drbd1552.out0 disk.drbd1550.out0 disk.drbd1285.out0 disk.drbd1931.out0 disk.drbd1828.out0 disk.drbd1938.out0) disk./.usagePercent32.34385607157447! disk.nvme0n1.out 51.958984375 disk.sda2.in0 disk.sda2.out 47.57421875 disk.drbd2041.out0 disk.drbd1226.in0 disk.drbd1356.out0 disk.drbd1289.out0 disk.dm-0.in 2.47265625 disk.drbd1226.out0 disk.drbd2118.in0 disk.drbd1356.in0 disk.drbd1550.in0 disk.drbd1938.in0# proc.memory.rssMB 11260.4765625 disk.sda.in0 disk.rbd1.out 7.1611328125 disk.drbd1673.in0 cpu13.9792592302555062 - _wandbruntime1764  _runtime1764qj  _step312 ' lr-AdamW/pg10.00010177436443047379 ' lr-AdamW/pg20.00010177436443047379 trainer/global_step3949 $ _timestamp1.785797052156261e+09 _runtime 1764.97923906M  $ _timestamp1.785797052156261e+09 _runtime 1764.97923906 _step312 ' lr-AdamW/pg10.00010177436443047379 ' lr-AdamW/pg20.00010177436443047379 trainer/global_step3949h#F2 - _wandbruntime1764  _runtime1764xx' O QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current249 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed67973935694848 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 trainer/global_step3949 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed67813239522304 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated15553547 % _timestamp1.7857970521572838e+09 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated26244429 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed26865735 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated11312439 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed15553545 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3569457365504 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed71543258191872 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 _runtime1764.980617518 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated71288635041792 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed71288530572800 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 3475378480128 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed71288530572800 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3466785493552 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated67813256561664 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated67962892 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3569322497024 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed67813239522304 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3475378480128 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed66128225341472 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated41718463 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current251 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated71543417964032 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated67973960598528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed41708807 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 _step313 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed26244427 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed26244427 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3475291050496 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated67813256561664 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated71288635041792 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated69595113677508 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed71288530572800 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 3475378480128 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed71288530572800 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3466785493552 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated67813256561664 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated67962892 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3569322497024 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed67813239522304 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3475378480128 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed66128225341472 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated41718463 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current251 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated71543417964032 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated67973960598528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed41708807 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 _step313 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed26244427 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed26244427 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3475291050496 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated67813256561664 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated71288635041792 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated69595113677508 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed67973935694848 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 trainer/global_step3949 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed67813239522304 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated15553547 % _timestamp1.7857970521572838e+09 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated26244429 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed26865735 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated11312439 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed15553545 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3569457365504 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed71543258191872 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 _runtime1764.980617518 Xw2 - _wandbruntime1765  _runtime1765tE M VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 % _timestamp1.7857970525428474e+09 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated67830423324672 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3570356285440 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated71306681431552 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed67830406285312 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed15557482 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated67991167715328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 trainer/global_step3949 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed67830406285312 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed71306576955392 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 149279232 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated41729000 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated41729000 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3476258106880 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated69612731323162 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3476170670080 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated15557484 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 3467662966774 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 _step314 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed41719330 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current252 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated11315241 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated26872725 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 =7DeviceStatsMonitor.on_train_batch_end/++qsegment.all.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 124375552 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated71306681431552 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 _runtime1765.366216827 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed67991142811648 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed26251070 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3570231909888 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed26251070 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 3476170670080 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated67980072 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3467748770010 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed26872471 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated26251072 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated26251072 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed67970400 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed71561374721536 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed69612628480566 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated67830423324672 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672q]& M TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated11315241 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated26872725 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 124375552 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated71306681431552 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 _runtime1765.366216827 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed67991142811648 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed26251070 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3570231909888 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed26251070 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 3476170670080 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated67980072 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3467748770010 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed26872471 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated26251072 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated26251072 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed67970400 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed71561374721536 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed69612628480566 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated67830423324672 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 % _timestamp1.7857970525428474e+09 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated67830423324672 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3570356285440 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated71306681431552 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed67830406285312 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed15557482 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated67991167715328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 trainer/global_step3949 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed67830406285312 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed71306576955392 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 149279232 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated41729000 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated41729000 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3476258106880 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated69612731323162 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3476170670080 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated15557484 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 3467662966774 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 _step314 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed41719330 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current252 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130Q2 - _wandbruntime1765  _runtime1765X"<  _step315 .loss/FRt2/detection_loss2.545613086436476 1loss/SRqqt2/detection_loss1.0792788545722116 &loss/total_loss26.006376865617923 % _timestamp1.7857970525444937e+09 0loss/FBt2/assignment_loss0.6783718228340149 /loss/FBt1/detection_loss2.8494771880513006 .loss/FBt2/detection_loss4.214561986923218 /loss/FRt1/assignment_loss3.876135434744493 /loss/FRt2/assignment_loss3.318948529447828 1loss/FBt1/assignment_loss0.21074154828631828 .loss/FRt1/detection_loss2.384668045448807 1loss/SRqqt1/detection_loss1.5219921090290978 trainer/global_step3949 2loss/SRqqt1/assignment_loss1.6366939221290833 1loss/SRqqt2/assignment_loss1.689894337715073 epoch0 _runtime1765.367597148  0loss/FBt2/assignment_loss0.6783718228340149 /loss/FBt1/detection_loss2.8494771880513006 .loss/FBt2/detection_loss4.214561986923218 /loss/FRt1/assignment_loss3.876135434744493 /loss/FRt2/assignment_loss3.318948529447828 1loss/FBt1/assignment_loss0.21074154828631828 .loss/FRt1/detection_loss2.384668045448807 1loss/SRqqt1/detection_loss1.5219921090290978 trainer/global_step3949 2loss/SRqqt1/assignment_loss1.6366939221290833 1loss/SRqqt2/assignment_loss1.689894337715073 epoch0 _runtime1765.367597148 _step315 .loss/FRt2/detection_loss2.545613086436476 1loss/SRqqt2/detection_loss1.0792788545722116 &loss/total_loss26.006376865617923 % _timestamp1.7857970525444937e+09 (: U gpu.0.gpu0.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts94.31' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.436666666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0@%R (:  U disk.drbd1965.out0 disk.drbd1931.in0 disk.drbd1510.in0 disk.drbd1552.in0 disk.rbd0.in0 disk.sda1.out0( proc.memory.availableMB 38516.859375 disk.drbd1817.in0 disk.dm-0.in 2.47265625# disk.nvme0n1.out52.29443359375 disk.drbd1356.in0 disk.drbd1938.in0 network.sent 3.662947e+06 disk.drbd1285.in0 disk.dm-0.out52.29443359375 disk.nvme0n1.in 2.47265625) disk./.usagePercent32.34385692492584 disk.drbd1673.in0 disk.sda2.out 48.171875 disk.drbd1584.in0 disk.rbd1.in0 disk.sda.out 48.171875 proc.cpu.threads29 disk.drbd1289.out0 disk.drbd1510.out0 disk.drbd2078.in0 disk.drbd2118.in0 disk.drbd2118.out0 disk.drbd1356.out0 disk.drbd1584.out0 disk.rbd0.out0 network.recv272604 disk.drbd1673.out0 disk.drbd1285.out0 disk.drbd1226.in0 disk.drbd1938.out0 disk.sda.in0 disk.drbd1817.out0 disk.rbd1.out 7.1611328125 disk.drbd1931.out0 disk.drbd2041.in0 disk.sda1.in0 disk.drbd2111.out0 disk.drbd2041.out0 disk.drbd1550.in0 disk.drbd1226.out0 disk.drbd2111.in0$ memory_percent41.22793674468994# proc.memory.rssMB 11260.4765625) proc.memory.percent17.18212366104126 disk.drbd1965.in0 disk.drbd2078.out0 cpu13.975479457577276 disk.drbd1828.in0 disk.drbd1828.out0 disk.drbd1550.out0 disk.drbd1289.in0 disk.sda2.in0$ disk./.usageGB578.3410110473633 disk.drbd1552.out0(@ (: гֻU gpu.0.gpu22.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts119.798' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent39.93266666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0 Q (:  гÉV disk.drbd2111.in0 disk.drbd1285.out0 disk.drbd1226.in0 disk.drbd2041.in0 disk.drbd1931.out0 disk.drbd1550.in0 disk.drbd1289.out0 disk.drbd1938.out0 disk.sda1.out0 disk.drbd2078.in0$ disk./.usageGB578.3410301208496 disk.rbd1.out 7.193359375 disk.dm-0.out 53.9599609375" disk.nvme0n1.out 53.9599609375 network.sent 3.663055e+06 disk.drbd1552.in0 disk.drbd1552.out0 disk.drbd1828.out0 disk.sda.in0 disk.drbd1584.in0 network.recv272712 disk.drbd1817.in0 disk.drbd1673.out0 disk.drbd1510.out0 disk.rbd0.out0* proc.memory.availableMB38516.30078125 disk.drbd1965.out0 disk.drbd2041.out0 disk.drbd1828.in0 disk.sda.out 48.609375 disk.drbd1226.out0 disk.drbd2118.in0 disk.rbd0.in0 disk.drbd1285.in0 disk.drbd1584.out0 disk.sda2.in0 disk.rbd1.in0 disk.drbd1673.in0 disk.drbd1931.in0 disk.drbd1938.in0 disk.drbd1510.in0 disk.drbd1356.in0) proc.memory.percent17.18212366104126 disk.sda2.out 48.609375 proc.cpu.threads29 disk.drbd1817.out0 disk.drbd1965.in0 disk.drbd2078.out0 disk.nvme0n1.in 2.47265625) disk./.usagePercent32.34385799161504 disk.drbd1550.out0 disk.sda1.in0# proc.memory.rssMB 11260.4765625 disk.drbd1356.out0 cpu13.972431571964167 disk.drbd2111.out0 disk.drbd2118.out0$ memory_percent41.22878909111023 disk.drbd1289.in0 disk.dm-0.in 2.47265625[Pz2 - _wandbruntime1787  _runtime17874  _runtime1787.039404619 _step316 & lr-AdamW/pg10.0001030629737547391 & lr-AdamW/pg20.0001030629737547391 trainer/global_step3999 % _timestamp1.7857970742164466e+09z  % _timestamp1.7857970742164466e+09 _runtime1787.039404619 _step316 & lr-AdamW/pg10.0001030629737547391 & lr-AdamW/pg20.0001030629737547391 trainer/global_step3999[2 - _wandbruntime1787  _runtime1787 Lj' O F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated11455803 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated72448934307328 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed26576577 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated68671594712064 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 _step317 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 _runtime1787.040631489 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed42236809 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed72448772438016 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed68671577672704 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current256 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed26576577 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 trainer/global_step3999 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated68823044 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3614617868800 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated66965250996832 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 3510759760184 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 % _timestamp1.7857970742174094e+09 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3519287185920 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 3519374615552 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed42236809 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated70476010757016 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated72190969327616 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated72190969327616 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated26576579 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed72190864858624 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3519374615552 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed72190864858624 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3510673957060 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated15750397 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated42246465 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed70475907914532 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed68671577672704 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated42246465 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated27206200 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed68834291534848 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 2537553928%a  O QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current256 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed26576577 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 trainer/global_step3999 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated68823044 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 KBDeviceStatsMonitor.on_train_batch_start/allocatio{A#n.large_pool.peak1291 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3614617868800 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated66965250996832 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 3510759760184 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 % _timestamp1.7857970742174094e+09 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3519287185920 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 3519374615552 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed42236809 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated70476010757016 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated72190969327616 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated72190969327616 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated26576579 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed72190864858624 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3519374615552 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed72190864858624 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3510673957060 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated15750397 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated42246465 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed70475907914532 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed68671577672704 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated42246465 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated27206200 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed68834291534848 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated11455803 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated72448934307328 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed26576577 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated68671594712064 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 _step317 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 _runtime1787.040631489 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed42236809 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed72448772438016 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed68671577672704 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632Owc2 - _wandbruntime1787  _runtime1787Vx& M WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed72466882651648 % _timestamp1.7857970745889444e+09 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed66981974129792 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3511637233518 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3520254242304 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 3511551430282 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated68840224 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed68830552 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed26583220 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 3520166805504 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 _runtime1787.412304559 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed68688744435712 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed68688744435712 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current254 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed68830552 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated26583222 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed42247332 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated27212946 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current252 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 3520254242304 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated26583222 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated11458612 _step318 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated68840224 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated42257002 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated66981991169152 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed70493525560074 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed26583220 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed68851498651648 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3615384000000 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 trainer/global_step3999 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated42257002 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed27212692 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3520166805504 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated72467042416640 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3615518861312 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated72209015717376 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated68688761475072xz& M A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated72467042416640 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3615518861312 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated72209015717376 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated68688761475072 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed72466882651648 % _timestamp1.7857970745889444e+09 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed66981974129792 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3511637233518 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3520254242304 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 3511551430282 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated68840224 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed68830552 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed26583220 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 3520166805504 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 _runtime1787.412304559 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed68688744435712 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed68688744435712 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current254 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed68830552 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated26583222 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed42247332 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated27212946 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current252 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 3520254242304 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated26583222 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated11458612 _step318 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated68840224 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated42257002 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated66981991169152 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed70493525560074 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed26583220 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed68851498651648 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3615384000000 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 trainer/global_step3999 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated42257002 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed27212692 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3520166805504 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak182 - _wandbruntime1787  _runtime1787 +  /loss/FRt1/assignment_loss4.030183520524398 1loss/FBt1/assignment_loss0.15292755758732376 /loss/FRt1/detection_loss2.1703672494577324 /loss/FRt2/detection_loss2.5036529914384156 1loss/SRqqt1/detection_loss1.6298958118764106 /loss/FBt2/detection_loss2.8404680490493774 % _timestamp1.7857970745906627e+09 2loss/SRqqt1/assignment_loss1.8287872964965886 epoch0 _runtime1787.413496277 /loss/FRt2/assignment_loss3.400589318342612 1loss/FBt2/assignment_loss0.42908242903649807 /loss/FBt1/detection_loss1.6694116735037308 %loss/total_loss23.31548825448206 _step319 2loss/SRqqt2/assignment_loss1.4702637080351513 0loss/SRqqt2/detection_loss1.189858649133821 trainer/global_step3999)޳   trainer/global_step3999 /loss/FRt1/assignment_loss4.030183520524398 1loss/FBt1/assignment_loss0.15292755758732376 /loss/FRt1/detection_loss2.1703672494577324 /loss/FRt2/detection_loss2.5036529914384156 1loss/SRqqt1/detection_loss1.6298958118764106 /loss/FBt2/detection_loss2.8404680490493774 % _timestamp1.7857970745906627e+09 2loss/SRqqt1/assignment_loss1.8287872964965886 epoch0 _runtime1787.413496277 /loss/FRt2/assignment_loss3.400589318342612 1loss/FBt2/assignment_loss0.42908242903649807 /loss/FBt1/detection_loss1.6694116735037308 %loss/total_loss23.31548825448206 _step319 2loss/SRqqt2/assignment_loss1.4702637080351513 0loss/SRqqt2/detection_loss1.189858649133821?M (:  ߳U disk.drbd1510.out0 disk.nvme0n1.in 2.47265625 disk.drbd1965.in0 disk.drbd1828.out0 disk.dm-0.out54.27001953125 disk.drbd1584.out0 disk.drbd2118.out0 disk.drbd1931.in0) proc.memory.availableMB 38517.1484375 disk.drbd2111.in0 disk.drbd2041.out0$ disk./.usageGB578.3410110473633 disk.drbd1938.out0 disk.dm-0.in 2.47265625 disk.rbd1.in0 disk.rbd1.out 7.287109375 disk.drbd1673.out0 disk.drbd1550.in0 cpu13.968976399060018 disk.sda.in0 disk.sda1.in0 disk.drbd1285.in0# proc.memory.rssMB 11260.4765625 disk.drbd1289.out0 network.sent 3.663121e+06 disk.drbd1510.in0 disk.drbd2078.in0 disk.drbd1817.in0 disk.rbd0.out0 disk.sda.out49.1875 disk.drbd1356.out0 disk.drbd1938.in0 disk.rbd0.in0 disk.sda1.out0 disk.drbd2118.in0 disk.drbd2111.out0 disk.drbd1673.in0 disk.drbd1552.out0 proc.cpu.threads29 network.recv272778) disk./.usagePercent32.34385692492584 disk.drbd1226.in0 disk.drbd1584.in0 disk.drbd1828.in0 disk.drbd1226.out0 disk.drbd1356.in0 disk.sda2.in0 disk.sda2.out49.1875 disk.drbd1931.out0 disk.drbd1817.out0 disk.drbd1550.out0) proc.memory.percent17.18212366104126 disk.drbd2078.out0# disk.nvme0n1.out54.27001953125 disk.drbd2041.in0 disk.drbd1289.in0 disk.drbd1965.out0 disk.drbd1552.in0# memory_percent41.2274956703186 disk.drbd1285.out0} (: ߳憋V gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts125.4' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent41.800000000000004 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0+MO - _wandbruntzDVime1809  _runtime1809TO  $ _timestamp1.785797096265539e+09 _runtime 1809.08854392 _step320 ' lr-AdamW/pg10.00010435158307900438 ' lr-AdamW/pg20.00010435158307900438 trainer/global_step4049  $ _timestamp1.785797096265539e+09 _runtime 1809.08854392 _step320 ' lr-AdamW/pg10.00010435158307900438 ' lr-AdamW/pg20.00010435158307900438 trainer/global_step40499J2 - _wandbruntime1809  _runtime1809p#3' O MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 3554648223692 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed27544349 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 3563370750976 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed15947245 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated69683196 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3563283321344 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated42774467 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed69529915823104 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed69694647374848 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed42764811 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated42774467 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3563283321344 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3554562420568 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed26908727 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed73093199144448 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3659789952512 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated69529932862464 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed69529915823104 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated11597389 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 $ _timestamp1.785797096266578e+09 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3563370750976 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated27544636 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed67802242573472 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 _runtime1809.089823105 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated67802259612832 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated26908729 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed42764811 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed69673538 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed26908727 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current285 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 trainer/global_step4049 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed73093199144448 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3659655084032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 ' O RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3563283321344 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated42774467 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed69529915823104 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed69694647374848 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed42764811 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated42774467 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3563283321344 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3554562420568 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed26908727 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed73093199144448 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3659789952512 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated69529932862464 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed69529915823104 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated11597389 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 $ _timestamp1.785797096266578e+09 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3563370750976 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated27544636 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed67802242573472 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 _runtime1809.089823105 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated67802259612832 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated26908729 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed42764811 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed69673538 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed26908727 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current285 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 trainer/global_step4049 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed73093199144448 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3659655084032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 3554648223692 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed27544349 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 3563370750976 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed15947245 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated69683196 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032Nӓ2 - _wandbruntime1809  _runtime1809-& M SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed67818982745792 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated15951184 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed27551100 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current250 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed73372412664320 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3660558172672 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ODeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed69547082586112 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed69690704 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed69690704 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated71374525482178 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated69700376 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated73111350003200 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated73372572429312 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated69547099625472 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated42785004 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed42775334 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated73111350003200 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated69547099625472 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed71374422639582 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed73111245527040 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3555525697026 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 trainer/global_step4049 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated26915372 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated67818999785152 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated69700376 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3660693033984 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed69547082586112 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 3564162940928 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated42785004 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed15951182 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated69711879395328 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3564162940928 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated11600168 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 _step322 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated69711879395328 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3564162940928 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated11600168 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 _step322 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed27551100 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current250 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed73372412664320 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3660558172672 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ODeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed69547082586112 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed69690704 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed69690704 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated71374525482178 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated69700376 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated73111350003200 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated73372572429312 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated69547099625472 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated42785004 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed42775334 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated73111350003200 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated69547099625472 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed71374422639582 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed73111245527040 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3555525697026 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 trainer/global_step4049 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated26915372 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated67818999785152 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated69700376 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3660693033984 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed69547082586112 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 3564162940928%f u2 - _wandbruntime1809  _runtime1809-  0loss/FRt2/assignment_loss3.3827618370976364 1loss/FBt1/assignment_loss0.08921396243991417 .loss/FRt2/detection_loss2.693517079478816 &loss/total_loss23.143507683234688 trainer/global_step4049 _runtime1809.466198307 1loss/SRqqt2/detection_loss1.5236036385921137 /loss/FRt1/assignment_loss4.000038145018405 1loss/FBt2/assignment_loss0.40126563457306474 .loss/FRt1/detection_loss2.142813457817328 /loss/FBt1/detection_loss1.5055109128237723 epoch0 _step323 2loss/SRqqt1/assignment_loss1.4576032252944247 2loss/SRqqt2/assignment_loss1.5057509371564408 0loss/SRqqt1/detection_loss1.553309608367272 /loss/FBt2/detection_loss2.8881192445755004 % _timestamp1.7857970966433601e+09sl   % _timestamp1.7857970966433601e+09 0loss/FRt2/assignment_loss3.3827618370976364 1loss/FBt1/assignment_loss0.08921396243991417 .loss/FRt2/detection_loss2.693517079478816 &loss/total_loss23.143507683234688 trainer/global_step4049 _runtime1809.466198307 1loss/SRqqt2/detection_loss1.5236036385921137 /loss/FRt1/assignment_loss4.000038145018405 1loss/FBt2/assignment_loss0.40126563457306474 .loss/FRt1/detection_loss2.142813457817328 /loss/FBt1/detection_loss1.5055109128237723 epoch0 _step323 2loss/SRqqt1/assignment_loss1.4576032252944247 2loss/SRqqt2/assignment_loss1.5057509371564408 0loss/SRqqt1/detection_loss1.553309608367272 /loss/FBt2/detection_loss2.8881192445755004$ (: U gpu.0.gpu37.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts129.862' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent43.287333333333336 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0G (:  ׂV network.sent 3.663539e+06 disk.drbd1828.in0 disk.drbd1938.out0 disk.sda2.in0 disk.drbd1965.out0 proc.cpu.threads29 disk.nvme0n1.in 2.47265625 disk.drbd1931.in0 disk.drbd2041.in0 disk.drbd1828.out0# disk.nvme0n1.out55.94580078125# disk./.usageGB578.336971282959! proc.memory.availableMB38516" memory_percent41.229248046875 disk.sda1.in0 disk.drbd1510.out0 disk.drbd1356.in0 disk.drbd1817.out0# proc.memory.rssMB 11260.4765625 disk.sda.out 49.640625 disk.drbd1584.out0 disk.rbd1.in0 cpu13.965763043568986 disk.drbd1356.out0 disk.drbd2111.out0) proc.memory.percent17.18212366104126 network.recv273833 disk.drbd1673.in0 disk.drbd1289.out0 disk.sda.in0 disk.dm-0.out55.94580078125 disk.drbd1226.out0 disk.drbd2078.in0 disk.drbd1550.in0 disk.sda1.out0 disk.drbd1285.in0 disk.drbd1285.out0 disk.drbd2078.out0 disk.drbd1673.out0 disk.drbd1938.in0 disk.sda2.out 49.640625 disk.drbd2118.in0 disk.drbd2118.out0 disk.drbd1552.out0 disk.drbd2111.in0 disk.drbd1931.out0 disk.drbd1550.out0 disk.drbd1510.in0 disk.drbd1584.in0 disk.rbd1.out 7.435546875 disk.rbd0.in0 disk.rbd0.out0 disk.drbd1226.in0 disk.drbd1817.in0 disk.drbd1965.in0) disk./.usagePercent32.34363100015237 disk.drbd2041.out0 disk.drbd1552.in0 disk.drbd1289.in0 disk.dm-0.in 2.47265625* (: ǑU gpu.0.gpu31.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts124.68' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent41.56 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0gU (:  U disk.dm-0.out56.10205078125 disk.drbd2118.in0 cpu13.965313106677172 disk.drbd1552.in0 disk.drbd1510.in0 disk.drbd2118.out0# disk.nvme0n1.out56.10205078125 disk.drbd1673.out0 disk.drbd1931.out0 disk.sda.out 50.32421875 disk.drbd2078.out0 disk.rbd1.in0 disk.drbd2078.in0 disk.drbd1584.out0 disk.sda2.in0 disk.drbd2111.out0 disk.drbd1289.in0) proc.memory.percent17.18212366104126 network.recv279576) disk./.usagePercent32.34354374497547 disk.rbd0.out0 disk.drbd1226.in0$ memory_percent41.22927188873291 disk.drbd1356.out0( proc.memory.availableMB 38515.984375 disk.drbd1817.out0 disk.drbd1965.in0 disk.drbd1285.out0 proc.cpu.threads29 disk.drbd1817.in0 disk.drbd1510.out0 disk.drbd1584.in0 disk.nvme0n1.in 2.47265625 disk.drbd1931.in0 disk.sda.in0 disk.sda2.out 50.32421875 disk.drbd1673.in0 disk.drbd1828.in0 disk.sda1.in0 network.sent 3.779993e+06$ disk./.usageGB578.3354110717773 disk.sda1.out0 disk.drbd1938.out0 disk.drbd1226.out0 disk.drbd2041.in0 disk.drbd1552.out0 disk.drbd1938.in0# proc.memory.rssMB 11260.4765625 disk.rbd1.out 7.435546875 disk.drbd1550.in0 disk.drbd1289.out0 disk.dm-0.in 2.47265625 disk.drbd2111.in0 disk.drbd1965.out0 disk.drbd2041.out0 disk.rbd0.in0 disk.drbd1285.in0 disk.drbd1356.in0 disk.drbd1828.out0 disk.drbd1550.out0m]2 - _wandbruntime1831  _runtime1831$l  ' lr-AdamW/pg20.00010564019240326969 trainer/global_step4099 % _timestamp1.7857971184791212e+09 _runtime1831.302094628 _step324 ' lr-AdamW/pg10.00010564019240326969a>"  ' lr-AdamW/pg10.00010564019240326969 ' lr-AdamW/pg20.00010564019240326969 trainer/global_step4099 % _timestamp1.7857971184791212e+09 _runtime1831.302094628 _step324sjƂ2 - _wandbruntime1831  _runtime1831H!8' O L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed27240877 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated70388271012864 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed27240877 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated27883818 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated72237804916032 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3598450884076 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated43302469 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated16144097 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed73995533430272 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 3607366886400 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed73995533430272 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 _runtime1831.303281355 _step325 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3704985347072 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 trainer/global_step4099 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated70543348 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated68639268228832 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed11739471 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed72237702073548 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed68639251189472 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed43292813 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3607279456768 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 3598536687200 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed70388253973504 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed70533690 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3607279456768 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current250 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated74260013465600 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3704854672896 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 $ _timestamp1.785797118480074e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated73995637899264 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed70555003214848 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated70388271012864 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed16144095 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated11739721 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated73995637899264 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed70388253973504 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated70555028118528 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated27240879 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3607366886400 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated27240879 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current252 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392#|  O L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated43302469 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated16144097 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed73995533430272 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 3607366886400 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed73995533430272 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 _runtime1831.303281355 _step325 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3704985347072 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 trainer/global_step4099 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated70543348 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated68639268228832 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed11739471 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed72237702073548 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed68639251189472 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed43292813 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3607279456768 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 3598536687200 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed70388253973504 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed70533690 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3607279456768 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current250 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated74260013465600 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3704854672896 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 $ _timestamp1.785797118480074e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated73995637899264 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed70555003214848 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated70388271012864 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed16144095 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated11739721 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated73995637899264 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed70388253973504 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated70555028118528 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 PCDeviceStatsL,U[FMonitor.on_train_batch_start/active.large_pool.allocated27240879 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3607366886400 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated27240879 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current252 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed27240877 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated70388271012864 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed27240877 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated27883818 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated72237804916032 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3598450884076_2 - _wandbruntime1831  _runtime1831Ƕֺ& M OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated27890587 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated74013684289024 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed27890337 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3705891992064 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 _step326 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated70560528 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 _runtime1831.668209071 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3705755033600 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated43313006 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed68655991361792 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 3599328357298 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated16148034 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed70550856 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed70572210331648 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed43303336 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 3608246513152 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated72255422561686 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed72255319719090 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current250 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3608159076352 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current248 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated70560528 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed74277965365248 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed70405420736512 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed16148032 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed70550856 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3608246513152 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated11742553 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated70405437775872 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated43313006 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed43303336 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated68656008401152 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3599414160534 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0<& M E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 _step326 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated70560528 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 _runtime1831.668209071 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3705755033600 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated43313006 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed68655991361792 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 3599328357298 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated16148034 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed70550856 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed70572210331648 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed43303336 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 3608246513152 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated72255422561686 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed72255319719090 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current250 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3608159076352 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current248 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated70560528 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed74277965365248 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed70405420736512 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed16148032 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed70550856 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3608246513152 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated11742553 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated70405437775872 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated43313006 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed43303336 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated68656008401152 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3599414160534 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated27890587 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated74013684289024 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed27890337 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3705891992064 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0C2 - _wandbruntime1831  _runtime1831'  .loss/FRt2/detection_loss2.936271364490191 1loss/SRqqt2/detection_loss1.2504520471604885 epoch0 /loss/FRt1/assignment_loss4.193315944156131 0loss/FRt2/assignment_loss3.4762637615203857 2loss/SRqqt1/assignment_loss1.5097199836944013 2loss/SRqqt2/assignment_loss1.5572679544964882 1loss/FBt1/assignment_loss0.05478385272325189 % _timestamp1.7857971188467932e+09 0loss/FBt2/assignment_loss0.4340682772919536 1loss/SRqqt1/detection_loss1.3840510210382329 _step327 /loss/FBt1/detection_loss2.2744131822966867 .loss/FBt2/detection_loss2.266458296775818 &loss/total_loss23.585064958817473 trainer/global_step4099 _runtime1831.669626582 .loss/FRt1/detection_loss2.247999273173444   /loss/FBt1/detection_loss2.2744131822966867 .loss/FBt2/detection_loss2.266458296775818 &loss/total_loss23.585064958817473 trainer/global_step4099 _runtime1831.669626582 .loss/FRt1/detection_loss2.247999273173444 .loss/FRt2/detection_loss2.936271364490191 1loss/SRqqt2/detection_loss1.2504520471604885 epoch0 /loss/FRt1/assignment_loss4.193315944156131 0loss/FRt2/assignment_loss3.4762637615203857 2loss/SRqqt1/assignment_loss1.5097199836944013 2loss/SRqqt2/assignment_loss1.5572679544964882 1loss/FBt1/assignment_loss0.05478385272325189 % _timestamp1.7857971188467932e+09 0loss/FBt2/assignment_loss0.4340682772919536 1loss/SRqqt1/detection_loss1.3840510210382329 _step327d+ (: U gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts125.071' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent41.690333333333335 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0e|M (:  U disk.drbd1550.in0) proc.memory.percent17.18212366104126 disk.drbd1938.in0 disk.drbd2118.out0 disk.drbd1552.in0 cpu13.962050316385946 disk.dm-0.out57.48388671875( disk./.usagePercent32.3435955860708 disk.drbd1584.in0 disk.sda2.in0 disk.drbd2078.out0 disk.sda2.out50.625 disk.drbd2111.out0 disk.drbd1828.in0 disk.drbd1289.out0% memory_percent41.227710247039795 network.sent 3.780784e+06 disk.drbd1931.in0 disk.drbd1289.in0 disk.drbd2118.in0 disk.sda.out50.625 disk.drbd2041.in0# disk.nvme0n1.out57.48388671875# proc.memory.rssMB 11260.4765625 disk.sda.in0 disk.drbd1817.out0 disk.drbd1673.in0 disk.drbd1931.out0 disk.rbd0.in0 disk.drbd1584.out0 disk.drbd1938.out0 disk.rbd0.out0 disk.drbd2078.in0 disk.rbd1.out 7.4677734375 disk.drbd1356.out0 disk.drbd2111.in0 disk.rbd1.in0 disk.drbd1552.out0 disk.drbd1285.in0 disk.drbd1510.in0 disk.dm-0.in 2.47265625 disk.nvme0n1.in 2.47265625 disk.drbd1965.in0 disk.drbd1965.out0$ disk./.usageGB578.3363380432129 disk.drbd1817.in0 disk.drbd1828.out0 disk.drbd1550.out0 disk.sda1.in0 disk.sda1.out0 disk.drbd1510.out0 disk.drbd1673.out0) proc.memory.availableMB 38517.0078125 disk.drbd1285.out0 disk.drbd1226.in0 disk.drbd1226.out0 disk.drbd1356.in0 proc.cpu.threads29 network.recv279708 disk.drbd2041.out0?2 - _wandbruntime1853  _runtime18535  _runtime1853.585931298 _step328 ' lr-AdamW/pg10.00010692880172753499 ' lr-AdamW/pg20.00010692880172753499 trainer/global_step4149 % _timestamp1.7857971407629764e+09$@  % _timestamp1.7857971407629764e+09 _runtime1853.585931298 _step328 ' lr-AdamW/pg10.00010692880172753499 ' lr-AdamW/pg20.00010692880172753499 trainer/global_step4149S2 - _wandbruntime1853  _runtime1853M,r O Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed74897867716096 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed27573027 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed43820815 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated28223315 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed71246592123904 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated73118701995540 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3750049727488 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3642339347584 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated27573029 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated71403500 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated11882368 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated16340947 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current225 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed73118599153056 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 trainer/global_step4149 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated75165558068736 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed69476259805472 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 _step329 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3651275592192 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 149286400 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed11882143 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed43820815 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed71415359054848 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated71415383958528 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 124382720tg' O dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated71415383958528 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 124382720 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed74897867716096 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed27573027 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed43820815 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated28223315 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed71246592123904 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated73118701995540 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3750049727488 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3642339347584 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated27573029 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated71403500 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated11882368 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated16340947 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current225 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed73118599153056 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 trainer/global_step4149 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated75165558068736 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed69476259805472 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 _step329 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3651275592192 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 149286400 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed11882143 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed43820815 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed71415359054848 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0F2 - _wandbruntime1853  _runtime1853F.q& M XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 3652155211776 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated75183675303936 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3652155211776 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed27579670 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated74916018574848 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated71263775926272 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated43841008 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated11885126 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed16344882 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current246 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 trainer/global_step4149 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed43831338 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated16344884 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated27579672 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated43841008 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed43831338 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed27579670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3652242648576 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated73136319641194 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed74915914098688 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed28229762 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed75183515538944 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 % _timestamp1.7857971411372907e+09 _step330 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3750949367296 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed69492999977792 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 3643216820806 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated71420680 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed71411008 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated28230010 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed11884880 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed71263758886912 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 3652242648576 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated27579672 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3751084228608 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed71432566171648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated74916018574848 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated71420680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated69493017017152 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed74915914098688 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed28229762 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed75183515538944 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 % _timestamp1.7857971411372907e+09 _step330 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3750949367296 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed69492999977792 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 3643216820806 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated71420680 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed71411008 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated28230010 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed11884880 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed71263758886912 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 3652242648576 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated27579672 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3751084228608 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed71432566171648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated74916018574848 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated71420680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated69493017017152 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed16344882 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current246 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 trainer/global_step4149 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed43831338 VIDeviceStatsMonitor.on_train_batch_endD/inactive_split.large_pool.allocated16344884 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated27579672 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated43841008 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed43831338 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121{2 - _wandbruntime1853  _runtime1853  0loss/FRt1/assignment_loss3.9813560017016756 .loss/FRt1/detection_loss2.237215401495204 .loss/FRt2/detection_loss2.738014846498316 1loss/SRqqt2/detection_loss1.3218111699399528 % _timestamp1.7857971411389065e+09 2loss/FBt1/assignment_loss0.018462964224580134 %loss/total_loss24.15268618500209 _step331 .loss/FBt2/detection_loss3.252954199910164 epoch0 0loss/FRt2/assignment_loss3.6168892242691735 2loss/SRqqt1/assignment_loss1.6918085927054995 /loss/FBt2/assignment_loss0.574257695581764 1loss/SRqqt1/detection_loss1.5875168684802996 _runtime 1853.96173442 2loss/SRqqt2/assignment_loss1.6443650902194136 /loss/FBt1/detection_loss1.4880341299760498 trainer/global_step4149  0loss/FRt1/assignment_loss3.9813560017016756 .loss/FRt1/detection_loss2.237215401495204 .loss/FRt2/detection_loss2.738014846498316 1loss/SRqqt2/detection_loss1.3218111699399528 % _timestamp1.7857971411389065e+09 2loss/FBt1/assignment_loss0.018462964224580134 %loss/total_loss24.15268618500209 _step331 .loss/FBt2/detection_loss3.252954199910164 epoch0 0loss/FRt2/assignment_loss3.6168892242691735 2loss/SRqqt1/assignment_loss1.6918085927054995 /loss/FBt2/assignment_loss0.574257695581764 1loss/SRqqt1/detection_loss1.5875168684802996 _runtime 1853.96173442 2loss/SRqqt2/assignment_loss1.6443650902194136 /loss/FBt1/detection_loss1.4880341299760498 trainer/global_step4149[ (: U gpu.0.gpu30.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts125.665' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent41.888333333333335 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0U (:  U disk.drbd1584.in0 disk.sda2.out 50.89453125 disk.drbd1938.in0) proc.memory.percent17.18212366104126 disk.drbd1817.out0 disk.drbd1965.in0 disk.drbd1828.in0 disk.dm-0.in 2.47265625 disk.drbd1584.out0 disk.nvme0n1.in 2.47265625 disk.drbd1965.out0 disk.drbd1552.out0& proc.memory.availableMB 38517.4375 disk.drbd1510.in0 disk.drbd1828.out0 disk.sda1.in0 disk.drbd2118.in0 disk.drbd1817.in0 disk.sda.out 50.89453125% memory_percent41.227054595947266 network.sent 3.848921e+06 disk.rbd1.in0 disk.rbd0.in0# proc.memory.rssMB 11260.4765625 cpu13.958974178042126 disk.drbd2041.out0 disk.drbd1289.out0 disk.drbd2111.out0 disk.drbd1931.out0 disk.drbd1285.out0 disk.drbd2078.out0 disk.drbd1552.in0 disk.rbd0.out0 disk.drbd2078.in0 disk.drbd1673.in0 disk.drbd1550.out0) disk./.usagePercent32.34401714164431 disk.sda1.out0 disk.drbd1510.out0 disk.drbd1356.out0 disk.drbd1673.out0 disk.drbd1289.in0 disk.drbd1938.out0 disk.dm-0.out58.10302734375 disk.drbd1226.in0 disk.drbd2118.out0 disk.drbd1285.in0 disk.drbd1356.in0 disk.drbd2111.in0 disk.drbd2041.in0 disk.drbd1550.in0 disk.sda.in0 network.recv282330 disk.drbd1226.out0# disk.nvme0n1.out58.10302734375 disk.sda2.in0 disk.rbd1.out 7.5615234375 disk.drbd1931.in0 proc.cpu.threads29$ disk./.usageGB578.34387588500981@ (: U gpu.0.gpu32.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts95.065' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.688333333333336 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0W (:  U disk.sda2.out 51.34765625 disk.rbd1.in0 disk.drbd1938.out0 disk.drbd1285.in0 disk.drbd1289.in0 disk.sda.in0) disk./.usagePercent32.34423559959334 disk.sda.out 51.34765625 disk.drbd1285.out0 disk.drbd1510.out0$ memory_percent41.22782349586487 disk.nvme0n1.in 2.47265625 disk.drbd1356.in0 disk.drbd1673.out0 disk.drbd2041.out0 disk.drbd1552.in0 disk.drbd1552.out0 disk.drbd2078.in0* proc.memory.availableMB38516.93359375 network.sent 3.848963e+06 network.recv282372$ disk./.usageGB578.3477821350098 disk.drbd1931.out0 disk.drbd2041.in0 disk.drbd2078.out0# proc.memory.rssMB 11260.4765625 disk.drbd1965.in0 disk.drbd1584.out0 disk.drbd2111.out0 disk.drbd1817.in0 disk.rbd1.out 7.6240234375 disk.dm-0.in 2.47265625) proc.memory.percent17.18212366104126 disk.drbd2118.out0 disk.drbd1817.out0 disk.drbd1828.out0 disk.drbd1289.out0 disk.drbd1226.in0 disk.drbd1938.in0 disk.drbd2118.in0 disk.drbd1828.in0 disk.rbd0.out0# disk.nvme0n1.out59.30126953125 disk.drbd2111.in0 disk.drbd1550.out0 disk.rbd0.in0 disk.sda1.in0 cpu13.95568557632311 proc.cpu.threads29 disk.drbd1584.in0 disk.drbd1356.out0 disk.drbd1931.in0 disk.drbd1550.in0 disk.sda1.out0 disk.drbd1226.out0 disk.dm-0.out59.30126953125 disk.drbd1965.out0 disk.sda2.in0 disk.drbd1673.in0 disk.drbd1510.in0-<2 - _wandbruntime1876  _runtime1876-!  & lr-AdamW/pg10.0001082174110518003 & lr-AdamW/pg20.0001082174110518003 trainer/global_step4199 $ _timestamp1.785797163196146e+09 _runtime1876.020699603 _step3320_  & lr-AdamW/pg10.0001082174110518003 & lr-AdamW/pg20.0001082174110518003 trainer/global_step4199 $ _timestamp1.785797163196146e+09 _runtime1876.020699603 _step332O2 - _wandbruntime1876  _runtime1876' O PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated27905179 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed44348817 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3695271727616 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed75800202001920 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated12024835 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current254 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated44358473 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed44348817 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 126479872 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3795243944960 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed73999496232564 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated75800306470912 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed28562376 _runtime1876.020797377 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 3686313614216 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed70313268421472 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed72275714894848 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed72104930274304 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed12024581 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3795370424832 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated72275739798528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated73999599075048 _step333 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated27905179 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated28562632 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated75800306470912 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3695359157248 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 3695359157248 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated76071110223360 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed27905177 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed75800202001920 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated16537797 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed76070958839808 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated70313285460832 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated72104947313664 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed72104930274304 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated72275739798528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated73999599075048 _step333 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated27905179 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated28562632 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated75800306470912 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3695359157248 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 3695359157248 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated76071110223360 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed27905177 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed75800202001920 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated16537797 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed76070958839808 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated70313285460832 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated72104947313664 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed72104930274304 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed73999496232564 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated75800306470912 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed28562376 _runtime1876.020797377 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 3686313614216 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed70313268421472 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed72275714894848 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121T %2 - _wandbruntime1876  _runtime1876d M K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed28569187 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed74017113878106 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated27911822 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed72122097037312 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3796278118400 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated16541734 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated76089225033728 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 3696238784000 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3696238784000 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed44359340 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed12027455 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3687191087550 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed16541732 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated70330025633152 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed72122097037312 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed44359340 WIDeviceStatsMonitor.on_train_baLȯqtch_end/reserved_bytes.small_pool.allocated 253755392 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated75818352860672 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed27911820 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 _runtime1876.384759232 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated75818352860672 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3696151347200 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed72292922011648 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated44369010 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current259 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated44369010 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated27911822 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated72122114076672 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 % _timestamp1.7857971635614257e+09 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated72280832 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _step334 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated72280832 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated72122114076672 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated12027712 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed76089067365888 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed72271160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 3696151347200 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887q?& M J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed27911820 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 _runtime1876.384759232 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated75818352860672 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3696151347200 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed72292922011648 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated44369010 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current259 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated44369010 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated27911822 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated72122114076672 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 % _timestamp1.7857971635614257e+09 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated72280832 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _step334 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated72280832 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated72122114076672 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated12027712 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed76089067365888 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed72271160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 3696151347200 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed28569187 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed74017113878106 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated27911822 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed72122097037312 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3796278118400 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated16541734 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated76089225033728 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 3696238784000 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3696238784000 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed44359340 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed12027455 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3687191087550 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed16541732 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated70330025633152 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed72122097037312 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed44359340 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated75818352860672 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak862781443b]2 - _wandbruntime1876  _runtime1876VϏ  0loss/FBt2/assignment_loss0.5929628871381283 epoch0 _runtime1876.385895172 /loss/FRt1/assignment_loss4.035024938376053 .loss/FBt1/detection_loss4.975200540454389 &loss/total_loss28.991410065137714 trainer/global_step4199 _step335 0loss/FRt2/assignment_loss3.2821446081687666 1loss/SRqqt2/assignment_loss1.778091893356759 .loss/FRt1/detection_loss2.495335636255534 0loss/SRqqt2/detection_loss1.561818831905839 1loss/SRqqt1/assignment_loss1.261978090786543 1loss/FBt1/assignment_loss0.11901524561658554 .loss/FRt2/detection_loss2.623766587212168 1loss/SRqqt1/detection_loss1.3893231619071618 .loss/FBt2/detection_loss4.876747643959783 $ _timestamp1.785797163563082e+09Z  &loss/total_loss28.991410065137714 trainer/global_step4199 _step335 0loss/FRt2/assignment_loss3.2821446081687666 1loss/SRqqt2/assignment_loss1.778091893356759 .loss/FRt1/detection_loss2.495335636255534 0loss/SRqqt2/detection_loss1.561818831905839 1loss/SRqqt1/assignment_loss1.261978090786543 1loss/FBt1/assignment_loss0.11901524561658554 .loss/FRt2/detection_loss2.623766587212168 1loss/SRqqt1/detection_loss1.3893231619071618 .loss/FBt2/detection_loss4.876747643959783 $ _timestamp1.785797163563082e+09 0loss/FBt2/assignment_loss0.5929628871381283 epoch0 _runtime1876.385895172 /loss/FRt1/assignment_loss4.035024938376053 .loss/FBt1/detection_loss4.975200540454389]| (: U gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts118.967' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent39.65566666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0%(IeS (:  U disk.dm-0.out65.12158203125 network.recv285939 disk.sda1.out0 disk.drbd1285.out0* disk./.usagePercent32.344191865335965 disk.drbd1552.in0 disk.sda.out51.84375 disk.drbd2118.out0 disk.drbd1356.in0 disk.drbd2078.out0 disk.drbd2111.out0 disk.drbd2078.in0 proc.cpu.threads29 disk.drbd1965.in0 disk.drbd2041.in0 disk.drbd2118.in0# disk.nvme0n1.out65.12158203125 disk.drbd2111.in0 cpu13.952510461532457 disk.drbd1673.in0 disk.drbd1931.in0 disk.nvme0n1.in 2.47265625 disk.drbd1817.in0* proc.memory.availableMB38516.90234375 disk.drbd1584.in0 disk.drbd1584.out0 disk.sda2.out51.84375# proc.memory.rssMB 11260.4765625 disk.drbd1828.in0 disk.drbd1289.out0 disk.sda.in0 disk.sda1.in0 disk.drbd1285.in0 disk.drbd1965.out0 disk.drbd1931.out0 disk.drbd1828.out0 disk.drbd1550.out0 disk.drbd1510.in0 network.sent 3.896819e+06$ disk./.usageGB578.3470001220703 disk.rbd1.out 7.6240234375 disk.drbd1938.in0 disk.rbd1.in0 disk.drbd1289.in0 disk.rbd0.out0 disk.drbd1510.out0 disk.drbd1226.in0 disk.drbd1226.out0 disk.sda2.in0$ memory_percent41.22787117958069 disk.drbd1817.out0 disk.drbd2041.out0 disk.drbd1550.in0 disk.drbd1938.out0 disk.rbd0.in0 disk.drbd1356.out0) proc.memory.percent17.18212366104126 disk.drbd1673.out0 disk.drbd1552.out0 disk.dm-0.in 2.47265625A&2 - _wandbruntime1897  _runtime18970h  % _timestamp1.7857971851120996e+09 _runtime 1897.93508842 _step336 ' lr-AdamW/pg10.00010950602037606561 ' lr-AdamW/pg20.00010950602037606561 trainer/global_step4249eG  trainer/global_step4249 % _timestamp1.7857971851120996e+09 _runtime 1897.93508842 _step336 ' lr-AdamW/pg10.00010950602037606561 ' lr-AdamW/pg20.00010950602037606561%j2 - _wandbruntime1897  _runtime1897?P' O \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated72963285464064 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated44886475 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current270 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed74880393312072 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed28237327 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated76976668396544 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated28901972 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed76702536287744 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated73123804 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3730116274600 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed12167057 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3840442083840 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated71150294076832 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed28237327 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3739355292672 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated76702640756736 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated73136095638528 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated72963285464064 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 _step337 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3739267863040 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 trainer/global_step4249 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed16734645 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated44886475 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated28237329 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated74880496154556 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 3739355292672 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed76976512818688 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated12167325 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856tVt  O TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed76702536287744 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3840572758016 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 3730202077724 _runtime1897.936354244 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed72963268424704 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3730116274600 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed12167057 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3840442083840 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated71150294076832 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed28237327 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3739355292672 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated76702640756736 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated73136095638528 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated72963285464064 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021"몸996032 _step337 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3739267863040 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 trainer/global_step4249 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed16734645 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated44886475 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated28237329 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated74880496154556 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 3739355292672 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed76976512818688 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated12167325 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated72963285464064 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated44886475 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current270 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed74880393312072 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed28237327 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated76976668396544 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated28901972 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed76702536287744 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated73123804 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed73131312 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated28243972 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 149279232 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated73140984 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 _step338 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated73140984 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated44897012 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated76994777760768 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3841475005440 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed72980435187712 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed73131312 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3740234919424 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 3740147482624 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated44897012 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed76720582670336 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed28908412 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed44887342 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated73153302755328 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3841350629888 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated72980452227072 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 trainer/global_step4249 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed76994628481536 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed71167017209792 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated16738584 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated71167034249152 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 3730993747822 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3731079551058 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024?v>& M J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed28243970 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed73131312 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated28243972 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 149279232 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated73140984 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 _step338 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated73140984 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated44897012 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated76994777760768 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3841475005440 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed72980435187712 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed73131312 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3740234919424 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 3740147482624 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated44897012 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed76720582670336 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed28908412 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed44887342 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated73153302755328 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3841350629888 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated72980452227072 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 trainer/global_step4249 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed76994628481536 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed71167017209792 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated16738584 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated71167034249152 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 3730993747822 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3731079551058 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed72980435187712 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed28243970 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed73153277851648 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed44887342 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 3740234919424 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated74898113800210 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3740147482624 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated28908645 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130x2 - _wandbruntime1898  _runtime1898  0loss/FRt1/assignment_loss3.8796464760326645 /loss/FRt2/assignment_loss3.353276878136855 2loss/SRqqt1/assignment_loss1.3504843388994534 1loss/SRqqt1/detection_loss1.7736287371876338 _step339 /loss/FRt2/detection_loss2.6157384692476344 1loss/SRqqt2/detection_loss1.1558263065186718 &loss/total_loss21.533454115438076 trainer/global_step4249 _runtime1898.308069988 2loss/SRqqt2/assignment_loss1.5925709234120755 1loss/FBt1/assignment_loss0.05072617213272679 /loss/FBt2/assignment_loss0.554847695309824 /loss/FRt1/detection_loss2.3642110749355796 /loss/FBt1/detection_loss1.9046827443594152 /loss/FBt2/detection_loss0.9378142992655436 epoch0 % _timestamp1.7857971854851768e+093b   _runtime1898.308069988 2loss/SRqqt2/assignment_loss1.5925709234120755 1loss/FBt1/assignment_loss0.05072617213272679 /loss/FBt2/assignment_loss0.554847695309824 /loss/FRt1/detection_loss2.3642110749355796 /loss/FBt1/detection_loss1.9046827443594152 /loss/FBt2/detection_loss0.9378142992655436 epoch0 % _timestamp1.7857971854851768e+09 0loss/FRt1/assignment_loss3.8796464760326645 /loss/FRt2/assignment_loss3.353276878136855 2loss/SRqqt1/assignment_loss1.3504843388994534 1loss/SRqqt1/detection_loss1.7736287371876338 _step339 /loss/FRt2/detection_loss2.6157384692476344 1loss/SRqqt2/detection_loss1.1558263065186718 &loss/total_loss21.533454115438076 trainer/global_step4249x (: ȴ¢U gpu.0.gpu23.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp40.0 gpu.0.powerWatts101.626' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent33.87533333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0DO (:  ȴU disk.drbd1938.in0 disk.drbd1285.in0 disk.drbd1552.in0 network.sent 3.897837e+06 disk.drbd1552.out0 disk.rbd1.in0 disk.drbd1550.out0 disk.drbd1226.in0! disk.nvme0n1.out 70.341796875 disk.sda2.in0$ memory_percent41.22791886329651 disk.drbd1828.out0 disk.sda.out 52.18359375 disk.drbd1285.out0 disk.dm-0.out 70.341796875 disk.drbd1965.out0) disk./.usagePercent32.34415239783541 disk.drbd1938.out0 disk.sda1.out0 disk.drbd1356.in0 disk.drbd2111.out0 disk.drbd1817.out0 disk.drbd1965.in0 disk.drbd2041.out0$ disk./.usageGB578.3462944030762 disk.drbd1828.in0 disk.drbd1289.out0 disk.drbd2078.in0 disk.drbd1584.out0 disk.drbd1931.out0 disk.drbd2041.in0 disk.drbd2118.in0 disk.rbd1.out7.65625# proc.memory.rssMB 11260.4765625 disk.sda.in0 disk.dm-0.in 2.47265625 disk.drbd2078.out0 disk.drbd1584.in0 disk.drbd2118.out0 disk.drbd2111.in0 disk.sda1.in0 disk.drbd1226.out0 disk.rbd0.out0 disk.drbd1673.out0 cpu13.949702815440297 disk.rbd0.in0 disk.drbd1817.in0 disk.drbd1673.in0 disk.drbd1931.in0 network.recv286835 disk.drbd1510.in0 disk.drbd1289.in0 disk.drbd1510.out0 disk.nvme0n1.in 2.47265625 disk.drbd1356.out0) proc.memory.percent17.18212366104126 disk.drbd1550.in0 disk.sda2.out 52.18359375* proc.memory.availableMB38516.87109375 proc.cpu.threads29d (: ״U gpu.0.gpu28.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp40.0 gpu.0.powerWatts97.423' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent32.474333333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0H (:  ״U network.recv288419 disk.drbd1931.in0 disk.drbd2041.in0( disk./.usagePercent32.3441513311462 disk.drbd1552.in0 disk.drbd1289.in0 disk.drbd1938.out0 disk.drbd1817.out0 cpu13.951970325419776) proc.memory.availableMB 38517.1171875 disk.drbd2041.out0 disk.drbd2118.out0! disk.nvme0n1.out 70.544921875 disk.rbd1.out7.75 disk.sda.out 52.5859375) proc.memory.percent17.18212366104126 disk.drbd1828.out0% memory_percent41.227543354034424 disk.drbd1285.out0 disk.drbd1550.in0 disk.rbd0.in0 disk.rbd0.out0 disk.dm-0.in 2.47265625 disk.drbd1226.in0 disk.drbd1584.out0 disk.drbd2111.in0 disk.drbd1552.out0 disk.sda.in0 disk.dm-0.out 70.544921875 disk.nvme0n1.in 2.47265625 disk.drbd1673.in0 disk.drbd1510.out0 disk.drbd2118.in0 disk.drbd1828.in0 disk.drbd1356.out0 disk.sda2.in0 disk.drbd2111.out0 disk.rbd1.in0 proc.cpu.threads29 disk.drbd1931.out0 disk.drbd1289.out0 disk.drbd1510.in0 disk.drbd1817.in0 disk.drbd1938.in0 disk.drbd1584.in0 disk.drbd1356.in0 disk.drbd1965.in0 disk.drbd1965.out0$ disk./.usageGB578.3462753295898 disk.drbd1673.out0 disk.drbd1285.in0 disk.drbd2078.in0 disk.drbd2078.out0 disk.sda2.out 52.5859375 network.sent 3.94261e+06 disk.drbd1550.out0 disk.sda1.in0 disk.sda1.out0 disk.drbd1226.out0# proc.memory.rssMB 11260.4765625bK2 - _wandbruntime1920  _runtime1920]  ' lr-AdamW/pg10.00011079462970033092 ' lr-AdamW/pg20.00011079462970033092 trainer/global_step4299 % _timestamp1.7857972079222367e+09 _runtime1920.745187525 _step340}_  _step340 ' lr-AdamW/pg10.00011079462970033092 ' lr-AdamW/pg20.00011079462970033092 trainer/global_step4299 % _timestamp1.7857972079222367e+09 _runtime1920.745187525'2 - _wandbruntime1920  _runtime1920T  O L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed28569477 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 37833514280965& trainer/global_step4299 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed73821606575104 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 3774090541232 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3774004738108 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed73996426574848 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed28569477 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed45404821 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated28569479 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated16931497 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed45404821 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated45414477 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated77882201212416 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated73996451478528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated12309687 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated71987302692832 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed71987285653472 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current262 _runtime1920.746391242 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed77604870573568 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current264 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed77604870573568 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated75761393234064 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 % _timestamp1.7857972079231467e+09 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed29240920 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 _step341 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated45414477 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3783351428096 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed73821606575104 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed16931495 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed77882039343104 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated73983956 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3885749733888 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated73821623614464 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3783263998464 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed12309425 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated77604975042560 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated73821623614464 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated28569479 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated29241184 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3783263998464 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130] ' O L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed28569477 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 3783351428096 trainer/global_step4299 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed73821606575104 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 3774090541232 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3774004738108 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed73996426574848 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed28569477 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed45404821 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated28569479 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated16931497 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed45404821 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated45414477 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated77882201212416 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated73996451478528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated12309687 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated71987302692832 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed71987285653472 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current262 _runtime1920.746391242 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed77604870573568 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current264 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed77604870573568 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated75761393234064 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 % _timestamp1.7857972079231467e+09 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed29240920 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 _step341 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated45414477 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3783351428096 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed73821606575104 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed16931495 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed77882039343104 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated73983956 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3885749733888 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated73821623614464 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3783263998464 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed12309425 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated77604975042560 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated73821623614464 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated28569479 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated29241184 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3783263998464 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak1302 - _wandbruntime1921  _runtime1921M& M [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated77900309053440 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3886650458112 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3774968014566 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed28576120 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated73838790377472 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed73838773338112 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated28576122 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated45425014 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed73838773338112 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated74001136 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3886519791104 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed73991464 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed28576120 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3784231054848 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current257 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed29247746 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 trainer/global_step4299 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated77623021432320 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated16935434 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated74013658595328 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 _runtime1921.115963591 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed74013633691648 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 3784231054848 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed77900153482752 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed16935432 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated75779010879718 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated73838790377472 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated28576122 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated29248003 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed72004025825792 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360>m  M OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed74013633691648 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 3784231054848 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed77900153482752 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed16935432 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated75779010879718 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated73838790377472 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated28576122 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated29248003 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/sl_egment.large_pool.current130 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed72004025825792 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated77900309053440 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3886650458112 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3774968014566 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed28576120 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated73838790377472 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed73838773338112 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated28576122 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated45425014 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed73838773338112 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated74001136 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3886519791104 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed73991464 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed28576120 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3784231054848 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current257 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed29247746 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 trainer/global_step4299 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated77623021432320 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated16935434 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated74013658595328 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 _runtime1921.115963591 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2sUh2 - _wandbruntime1921  _runtime1921b-Y  /loss/FRt1/assignment_loss4.057267621298816 0loss/FRt2/assignment_loss2.8958557031371375 2loss/SRqqt1/assignment_loss1.4852351397275925 0loss/FBt1/assignment_loss0.0435848476834176 0loss/FBt2/assignment_loss0.8072067218832671 .loss/FRt1/detection_loss2.454973958641569 -loss/FBt1/detection_loss1.30958926682897 %loss/total_loss28.42183430157139 .loss/FBt2/detection_loss7.513499826937914 epoch0 % _timestamp1.7857972082943616e+09 _step343 /loss/FRt2/detection_loss2.6055532734502447 _runtime1921.117192101 2loss/SRqqt2/assignment_loss1.7290399555976574 0loss/SRqqt1/detection_loss2.488538067255701 0loss/SRqqt2/detection_loss1.031489919129104 trainer/global_step4299dc  .loss/FRt1/detection_loss2.454973958641569 -loss/FBt1/detection_loss1.30958926682897 %loss/total_loss28.42183430157139 .loss/FBt2/detection_loss7.513499826937914 epoch0 % _timestamp1.7857972082943616e+09 _step343 /loss/FRt2/detection_loss2.6055532734502447 _runtime1921.117192101 2loss/SRqqt2/assignment_loss1.7290399555976574 0loss/SRqqt1/detection_loss2.488538067255701 0loss/SRqqt2/detection_loss1.031489919129104 trainer/global_step4299 /loss/FRt1/assignment_loss4.057267621298816 0loss/FRt2/assignment_loss2.8958557031371375 2loss/SRqqt1/assignment_loss1.4852351397275925 0loss/FBt1/assignment_loss0.0435848476834176 0loss/FBt2/assignment_loss0.8072067218832671B (: ´U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp40.0 gpu.0.powerWatts123.158' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent41.05266666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0O&0cN (:  U disk.drbd2111.in0 disk.drbd2111.out0 disk.drbd1289.out0 disk.drbd1817.in0 disk.rbd0.out0 disk.drbd1226.out0 disk.drbd1828.out0 disk.drbd1289.in0 disk.drbd1938.in0 disk.rbd0.in0 disk.drbd1226.in0 disk.drbd2118.out0* proc.memory.availableMB38516.99609375 disk.dm-0.out71.34814453125 disk.drbd1965.in0 network.sent 3.989681e+06 disk.sda1.in0 disk.drbd1584.in0 disk.drbd1552.in0 disk.drbd1552.out0 disk.drbd1550.out0 disk.sda.in0 disk.drbd1285.in0 disk.drbd2078.out0 disk.drbd1673.in0 disk.sda2.out 53.171875 disk.drbd1817.out0# proc.memory.rssMB 11260.4765625 disk.dm-0.in 2.47265625 disk.drbd2078.in0 disk.drbd2118.in0 disk.drbd2041.in0 disk.drbd2041.out0 disk.drbd1828.in0 disk.drbd1550.in0 cpu13.953327124496768 disk.drbd1356.in0 disk.drbd1356.out0 proc.cpu.threads29$ disk./.usageGB578.3462181091309 disk.rbd1.out7.8125# disk.nvme0n1.out71.34814453125 disk.rbd1.in0 disk.drbd1673.out0 disk.drbd1931.in0 disk.sda.out 53.171875 disk.drbd1584.out0 disk.nvme0n1.in 2.47265625 disk.sda2.in0$ memory_percent41.22772812843323 network.recv290853 disk.sda1.out0 disk.drbd1285.out0 disk.drbd1510.in0 disk.drbd1510.out0 disk.drbd1965.out0 disk.drbd1931.out0) proc.memory.percent17.18212366104126) disk./.usagePercent32.34414813107859 disk.drbd1938.out0P2 - _wandbruntime1943  _runtime1943f!  _runtime1943.241847578 _step344 ' lr-AdamW/pg10.00011208323902459621 ' lr-AdamW/pg20.00011208323902459621 trainer/global_step4349 % _timestamp1.7857972304188697e+09  % _timestamp1.7857972304188697e+09 _runtime1943.241847578 _step344 ' lr-AdamW/pg10.00011208323902459621 ' lr-AdamW/pg20.00011208323902459621 trainer/global_step4349'ɤ?2 - _wandbruntime1943  _runtime1943-nm' O KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed45932823 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed28901627 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated76642290313572 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 3827347563520 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated28901629 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed74679944725504 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated74844108 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated17128347 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated74856807318528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated45942479 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed17128345 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3817893201616 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed12453884 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated29582464 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed78507204859392 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated12454117 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 trainer/global_step4349 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3930761280000 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3827260133888 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated78507309328384 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current235 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated45942479 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed29582229 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed76642187471088 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 _step345 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3827347563520 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3930898245632 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 % _timestamp1.7857972304199104e+09 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated78787705564160 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed74856782414848 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed72824294269472 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3827260133888 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated72824311308832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 _runtime1943.243123431 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated78507309328384 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated74679961764864 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated28901629 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3827260133888 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated72824311308832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 _runtime1943.243123431 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated78507309328384 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated74679961764864 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated28901629 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated17128347 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated74856807318528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated45942479 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed17128345 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3817893201616 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed12453884 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated29582464 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed78507204859392 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated12454117 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 trainer/global_step4349 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3930761280000 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3827260133888 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated78507309328384 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current235 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated45942479 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed29582229 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0e)2 - _wandbruntime1943  _runtime1943!u M D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated28908272 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freedS#0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed74697111488512 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed29589059 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current263 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed78525251241984 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated45953016 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed72841034441792 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated74697128527872 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed45943346 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed74851616 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3818856478074 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 3818770674838 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 _step346 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed76659805116630 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _runtime1943.613619432 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed12456777 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed28908270 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated72841051481152 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 trainer/global_step4349 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed28908270 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated78805830680064 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated29589324 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 % _timestamp1.7857972307902646e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 3828227190272 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 3828139753472 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed74851616 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated28908272 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated76659907959226 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated17132284 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated45953016 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed45943346 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated74861288 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated74861288 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3828139753472 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated12457040 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3931816244736 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated74697128527872 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3828227190272 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated78525355718144 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed74697111488512 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed74873989531648 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121In& M UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 % _timestamp1.7857972307902646e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 3828227190272 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 3828139753472 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed74851616 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated28908272 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated76659907959226 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated17132284 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated45953016 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed45943346 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated74861288 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated74861288 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3828139753472 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated12457040 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 3931816244736 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated74697128527872 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3828227190272 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated78525355718144 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed74697111488512 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed74873989531648 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated28908272 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed74697111488512 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed29589059 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current263 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed78525251241984 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated45953016 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed72841034441792 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated74697128527872 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed45943346 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed74851616 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3818856478074 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 3818770674838 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 _step346 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed76659805116630 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _runtime1943.613619432 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed12456777 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed28908270 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated72841051481152 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 trainer/global_step4349 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed28908270 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated78805830680064 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated29589324^[2 - _wandbruntime1943  _runtime1943V  trainer/global_step4349 _runtime1943.614793217 0loss/FRt2/assignment_loss3.2538896401723227 1loss/SRqqt1/assignment_loss1.605630780521192 % _timestamp1.7857972307919483e+09 _step347 2loss/SRqqt2/assignment_loss1.5579963665509569 -loss/FRt2/detection_loss2.52741711779877 1loss/SRqqt2/detection_loss1.3456078779181817 %loss/total_loss37.14269505955816 0loss/FBt1/assignment_loss0.0719633205199999 0loss/FBt2/assignment_loss0.4777114274246352 -loss/FRt1/detection_loss2.32395213280084 /loss/FBt2/detection_loss3.3007846730095998 epoch0 /loss/FRt1/assignment_loss4.113106912037112 1loss/SRqqt1/detection_loss13.763579560737861 /loss/FBt1/detection_loss2.8010552500666868ө  trainer/global_step4349 _runtime1943.614793217 0loss/FRt2/assignment_loss3.2538896401723227 1loss/SRqqt1/assignment_loss1.605630780521192 % _timestamp1.7857972307919483e+09 _step347 2loss/SRqqt2/assignment_loss1.5579963665509569 -loss/FRt2/detection_loss2.52741711779877 1loss/SRqqt2/detection_loss1.3456078779181817 %loss/total_loss37.14269505955816 0loss/FBt1/assignment_loss0.0719633205199999 0loss/FBt2/assignment_loss0.4777114274246352 -loss/FRt1/detection_loss2.32395213280084 /loss/FBt2/detection_loss3.3007846730095998 epoch0 /loss/FRt1/assignment_loss4.113106912037112 1loss/SRqqt1/detection_loss13.763579560737861 /loss/FBt1/detection_loss2.8010552500666868Oќ (: ҆U gpu.0.gpu21.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp40.0 gpu.0.powerWatts127.147' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent42.382333333333335 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0R (:  ȺU network.recv290919 disk.drbd1356.out0 disk.drbd1965.in0 disk.drbd1965.out0 disk.drbd1938.in0$ memory_percent41.22830629348755 disk.drbd1938.out0 disk.drbd1817.in0 disk.drbd1817.out0 disk.drbd1226.in0 disk.drbd2078.in0 disk.drbd1584.out0 disk.drbd1356.in0 disk.drbd2118.in0 disk.drbd2041.in0 disk.drbd2041.out0) proc.memory.percent17.18212366104126 proc.cpu.threads29 disk.drbd1828.out0 disk.sda1.in0 disk.drbd1285.in0 cpu13.955724255370848 disk.drbd2111.out0 disk.rbd1.in0 disk.drbd1552.in0 disk.drbd1828.in0 disk.drbd1289.in0 disk.drbd1285.out0 disk.dm-0.out71.60986328125 disk.drbd2118.out0 disk.rbd0.in0 disk.drbd1510.out0 disk.drbd1550.in0 disk.sda.in0 disk.nvme0n1.in 2.47265625 disk.rbd1.out7.8125 disk.drbd1931.out0) proc.memory.availableMB 38516.6171875 disk.drbd2078.out0 disk.sda2.in0 disk.drbd1931.in0 disk.rbd0.out0$ disk./.usageGB578.3466644287109 disk.drbd1673.out0# proc.memory.rssMB 11260.4765625 network.sent 3.989747e+06 disk.drbd1584.in0 disk.drbd2111.in0 disk.drbd1673.in0 disk.drbd1226.out0# disk.nvme0n1.out71.60986328125 disk.sda2.out 53.72265625 disk.dm-0.in 2.47265625 disk.drbd1550.out0* disk./.usagePercent32.344173091605974 disk.drbd1289.out0 disk.sda.out 53.72265625 disk.drbd1510.in0 disk.drbd1552.out0 disk.sda1.out0dZ (: U gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp40.0 gpu.0.powerWatts127.435' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.47833333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0~Aq X (:  ΪU disk.drbd1356.out0 disk.drbd1931.out0* disk./.usagePercent32.344174158295175 disk.drbd2041.in0 disk.drbd2041.out0 disk.drbd1552.in0 disk.drbd1289.in0 disk.drbd1938.out0 disk.sda.in0 disk.drbd1510.in0 disk.drbd1226.in0 disk.drbd2078.in0 disk.drbd2118.in0 disk.nvme0n1.in 2.47265625 disk.drbd1552.out0 disk.dm-0.out71.80517578125 disk.rbd1.in0 disk.drbd1226.out0 disk.drbd2078.out0 disk.dm-0.in 2.47265625 disk.drbd1938.in0 disk.drbd1285.out0 disk.drbd1584.out0 disk.drbd1673.out0) proc.memory.availableMB 38516.6015625 network.recv293302 disk.drbd2118.out0 disk.drbd2111.in0 disk.drbd1817.in0$ disk./.usageGB578.3466835021973 disk.drbd1817.out0 disk.drbd1828.in0 disk.rbd1.out 7.8447265625 disk.drbd1673.in0 disk.drbd1550.in0 disk.drbd1965.in0 disk.drbd1931.in0 network.sent 4.037259e+06 disk.drbd1289.out0 disk.sda1.out0# disk.nvme0n1.out71.80517578125 disk.drbd1356.in0 disk.sda2.out 54.14453125 disk.sda2.in0 disk.drbd2111.out0 disk.drbd1828.out0 disk.rbd0.in0 disk.drbd1285.in0 disk.drbd1584.in0) proc.memory.percent17.18212366104126 disk.sda1.in0 disk.drbd1510.out0 disk.drbd1965.out0 cpu13.953649907531263 disk.rbd0.out0 disk.sda.out 54.14453125$ memory_percent41.22833013534546# proc.memory.rssMB 11260.4765625 proc.cpu.threads29 disk.drbd1550.out02 - _wandbruntime1965  _runtime1965x#  ' lr-AdamW/pg10.00011337184834886152 ' lr-AdamW/pg20.00011337184834886152 trainer/global_step4399 # _timestamp1.78579725236734e+09 _runtime1965.190306228 _step348Ta  _step348 ' lr-AdamW/pg10.00011337184834886152 ' lr-AdamW/pg20.00011337184834886152 trainer/global_step4399 # _timestamp1.78579725236734e+09 _runtime1965.190306228EN2 - _wandbruntime1965  _runtime1965q O L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated29923646 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed79693053406208 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed46460825 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated79409643614208 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated73661319924832 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated77523187393080 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 3871343698944 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated29233779 % _timestamp1.7857972523682854e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed79409539145216 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed77523084550596 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed73661302885472 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed75694602 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3871256269312 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3976047922688 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _step349 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated75704260 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current244 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed75538282875904 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current246 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3975915151360 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed29233777 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated12598449 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated79693211081216 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed79409539145216 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed29923400 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated29233779 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated79409643614208 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3871343698944 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3871256269312 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251a' O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed79409539145216 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed77523084550596 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed73661302885472 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed75694602 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3871256269312 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 3976047922688 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _step349 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated75704260 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current244 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed75538282875904 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current246 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 3975915151360 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed29233777 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated12598449 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated79693211081216 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed79409539145216 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed29923400 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated29233779 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated79409643614208 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3871343698944 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3871256269312 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated29923646 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed79693053406208 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed46460825 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated79409643614208 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated73661319924832 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated77523187393080 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 3871343698944 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated29233779 % _timestamp1.7857972523682854e+09$2 - _wandbruntime1965  _runtime1965UE& M E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _step350 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated79711319976960 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed73678043057792 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated75555466678272 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed75734345371648 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 % _timestamp1.7857972527315507e+09 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed75555449638912 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated75721440 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated46481018 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed29930147 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated29240422 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated12601243 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed12601015 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3976819034624 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 _runtime1965.554907241 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated75721440 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated79427690003968 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current230 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current228 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed75711768 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated79427690003968 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3862744941582 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3872223325696 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated29930377 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed17329132 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated75734370275328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed75555449638912 trainer/global_step4399 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed79427585527808 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed29240420 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 3872223325696 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed46471348 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 3862659138346 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed75555449638912 trainer/global_step4399 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed79427585527808 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed29240420 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 3872223325696 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed46471348 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 3862659138346 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 % _timestamp1.7857972527315507e+09 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed75555449638912 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated75721440 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated46481018 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed29930147 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated29240422 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated12601243 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed12601015 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 3976819034624 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 _runtime1965.554907241 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated75721440 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated79427690003968 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current230 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current228 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed75711768 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated79427690003968 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3862744941582 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3872223325696 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated29930377 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed17329132 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated75734370275328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 EDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed80311873431040 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated46998483 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated76577518998528 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3915339834368 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed80311873431040 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed29565927 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 trainer/global_step4449 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed29565927 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 3905755931756 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current246 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 4021094370304 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated76396638065664 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated30262723 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated76564412 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated80598750334464 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated78404084472588 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated80311977900032 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated29565929 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed76396621026304 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3915252404736 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed76396621026304 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current244 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed74498311501472 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed30262477 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3905670128632 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated46998483 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed80598588465152 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3915252404736 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated76396638065664 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated74498328540832 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed76577494094848 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130q{' O [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 3915339834368 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed80311873431040 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed29565927 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 trainer/global_step4449 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed29565927 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 3905755931756 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current246 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 4021094370304 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated76396638065664 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated30262723 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated76564412 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated80598750334464 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated78404084472588 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated80311977900032 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated29565929 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed76396621026304 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 3915252404736 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed76396621026304 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current244 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed74498311501472 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed30262477 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 3905670128632 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated46998483 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed80598588465152 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3915252404736 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated76396638065664 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated74498328540832 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed76577494094848 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 _step353 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed76554754 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed12740432 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed80311873431040 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated46998483 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated765775189985289N2 - _wandbruntime1987  _runtime1987. M D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated80330024289792 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated76581592 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated78421702118242 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed30269219 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated80616857427968 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated29572572 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated76413804828672 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3916132024320 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed80329919813632 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated47009020 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current254 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed76413787789312 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _step354 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated80330024289792 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed76571920 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current252 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed29572570 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated74515068713152 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed17525982 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed29572570 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 % _timestamp1.7857972747786264e+09 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed12743237 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed78421599275646 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated30269473 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 3916132024320 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed76571920 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated12743489 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed74515051673792 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed80616697662976 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed46999350 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated76413804828672 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4022131312640 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 3916219461120 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3916219461120 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated29572572 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed46999350 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed76594701211648 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 % _timestamp1.7857972747786264e+09 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed12743237 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed78421599275646 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated30269473 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 3916132024320 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed76571920 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated12743489 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed74515051673792 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed80616697662976 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed46999350 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated76413804828672 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4022131312640 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 3916219461120 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3916219461120 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated29572572 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed46999350 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed76594701211648 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated80330024289792 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated76581592 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated78421702118242 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed30269219 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated80616857427968 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated29572572 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated76413804828672 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3916132024320 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed80329919813632 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated47009020 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current254 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed76413787789312 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _step354 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated80330024289792 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed76571920 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current252 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated2512 - _wandbruntime1987  _runtime1987XDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated29898079 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated77254976216064 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed47516829 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current256 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed77437849934848 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated75335337156832 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated30602272 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 % _timestamp1.7857972962461817e+09 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated47526485 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed77254959176704 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4066406011392 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed17718895 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed81214207716864 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed30602014 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed81214207716864 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated77437874838528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed77414906 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 3949644395264 _step357 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated77254976216064 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed47516829 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed75335320117472 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated81504280849920 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 4066275337216 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated29898079 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 3959248540160 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated29898079 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated77254976216064 G>DeviceStatsMonitorzj .on_train_batch_start/active.large_pool.peak1291 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed47516829 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current256 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed77437849934848 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated75335337156832 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated30602272 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 % _timestamp1.7857972962461817e+09 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated47526485 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed77254959176704 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4066406011392 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed17718895 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed81214207716864 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed30602014 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed81214207716864 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated77437874838528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed77414906 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 3949644395264 _step357 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated77254976216064 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251_ 2 - _wandbruntime2009  _runtime2009kH& M bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated77455081955328 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 3950436065362 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed77272125939712 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 4067185217024 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed77432072 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed77455057051648 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed79302496355154 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated77441744 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3960215596544 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated81232358575616 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed81232254099456 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed29904720 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated77441744 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed12885945 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed47527352 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed77272125939712 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 118084096 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated17722834 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed30608777 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 trainer/global_step4499 _runtime2009.446057566 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current236 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated81522385256448 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated77272142979072 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated75352077329152 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed77432072 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 142987776 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 3960128159744 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current234 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3950521868598 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated30609013 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed17722832 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4067303301120 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated81232358575616 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated12886179 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated29904722 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 3960215596544 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 % _timestamp1.7857972966227257e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated47537022 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3960128159744 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed47527352 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated77272142979072 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated47537022 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed75352060289792 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed81522242268672 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated29904722 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated79302599197750 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 3950521868598 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated30609013 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed17722832 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4067303301120 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated81232358575616 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated12886179 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated29904722 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 3960215596544 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 % _timestamp1.7857972966227257e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated47537022 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 3960128159744 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed47527352 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated77272142979072 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated47537022 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed75352060289792 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed81522242268672 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated29904722 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated79302599197750 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 3950436065362 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed77272125939712 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 4067185217024 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed77432072 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed77455057051648 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed79302496355154 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated77441744 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 3960215596544 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated81232358575616 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed81232254099456 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed29904720 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated77441744 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed12885945 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed47527352 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed77272125939712 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 118084096 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated17722834 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed30608777 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 trainer/global_step4499 _runtime2009.446057566 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current236 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated81522385256448 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated77272142979072 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated75352077329152 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed77432072 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 142987776#4r2 - _wandbruntime2009  _runtime2009r[  1loss/SRqqt2/detection_loss0.9408666265251882 epoch0 /loss/FRt1/assignment_loss3.950587596744299 2loss/SRqqt2/assignment_loss1.4434649345629356 .loss/FRt1/detection_loss2.149927622405812 1loss/SRqqt1/detection_loss2.9215590966942493 /loss/FBt2/detection_loss1.5887780199118424 trainer/global_step4499 $ _timestamp1.785797296624621e+09 _step359 /loss/FRt2/assignment_loss3.114139393965403 2loss/SRqqt1/assignment_loss1.8426499482658174 1loss/FBt1/assignment_loss0.10494642240843594 .loss/FRt2/detection_loss2.700391265253226 &loss/total_loss23.104858292462403 _runtime2009.447451326 0loss/FBt2/assignment_loss0.5448475017389379 /loss/FBt1/detection_loss1.8026998639862575rǣ  1loss/SRqqt2/detection_loss0.9408666265251882 epoch0 /loss/FRt1/assignment_loss3.950587596744299 2loss/SRqqt2/assignment_loss1.4434649345629356 .loss/FRt1/detection_loss2.149927622405812 1loss/SRqqt1/detection_loss2.9215590966942493 /loss/FBt2/detection_loss1.5887780199118424 trainer/global_step4499 $ _timestamp1.785797296624621e+09 _step359 /loss/FRt2/assignment_loss3.114139393965403 2loss/SRqqt1/assignment_loss1.8426499482658174 1loss/FBt1/assignment_loss0.10494642240843594 .loss/FRt2/detection_loss2.700391265253226 &loss/total_loss23.104858292462403 _runtime2009.447451326 0loss/FBt2/assignment_loss0.5448475017389379 /loss/FBt1/detection_loss1.8026998639862575\ʏ (: ⠐U gpu.0.gpu13.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp41.0 gpu.0.powerWatts96.081' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent32.027 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0U (:  U network.recv294473 disk.drbd1550.in0 disk.sda.in0 disk.drbd1673.out0 disk.drbd1817.out0 disk.drbd1965.in0 disk.drbd1931.out0 cpu13.944963695233874 disk.drbd1226.out0 network.sent 4.037917e+06$ disk./.usageGB578.3516845703125 disk.drbd1289.in0 disk.drbd1356.in0* proc.memory.availableMB38516.37109375 disk.drbd1828.out0 disk.sda2.in0 disk.rbd1.in0 disk.drbd1938.in0 disk.rbd0.out0 disk.sda1.out0! disk.nvme0n1.out 72.873046875 disk.drbd2111.in0 disk.rbd1.out 8.0009765625% memory_percent41.228681802749634) proc.memory.percent17.18212366104126 disk.drbd1289.out0 disk.sda.out 55.21484375 disk.drbd2041.out0 proc.cpu.threads29 disk.drbd2078.out0 disk.drbd1285.in0 disk.drbd1817.in0 disk.drbd2111.out0 disk.drbd1965.out0 disk.drbd1931.in0 disk.drbd2041.in0 disk.drbd1552.in0 disk.rbd0.in0 disk.drbd1285.out0 disk.drbd1226.in0 disk.sda2.out 55.21484375# proc.memory.rssMB 11260.4765625 disk.drbd1938.out0 disk.nvme0n1.in 2.47265625 disk.drbd1673.in0) disk./.usagePercent32.34445384420453 disk.drbd1828.in0 disk.sda1.in0 disk.drbd1356.out0 disk.dm-0.out 72.873046875 disk.drbd1584.out0 disk.drbd1510.in0 disk.drbd2078.in0 disk.drbd2118.out0 disk.drbd2118.in0 disk.drbd1552.out0 disk.drbd1550.out0 disk.drbd1584.in0 disk.drbd1510.out0 disk.dm-0.in 2.47265625g  (: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp41.0 gpu.0.powerWatts100.935' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent33.645 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0ppDU (:  U$ disk./.usageGB578.3502006530762 disk.drbd2041.out0 disk.drbd1550.out0 disk.rbd0.out0) proc.memory.percent17.18212366104126 cpu13.942311138687229 disk.drbd2078.in0 disk.drbd2078.out0 disk.rbd0.in0 disk.drbd2118.out0 disk.drbd1817.in0 disk.drbd1965.out0 disk.drbd1673.in0 disk.drbd1552.out0 proc.cpu.threads29 disk.drbd1226.in0$ memory_percent41.22869372367859 disk.drbd1356.out0 disk.drbd1965.in0 disk.drbd1289.out0 disk.drbd1510.out0 network.recv296616 disk.sda2.out 55.49609375 disk.drbd1938.in0 disk.sda1.out0# proc.memory.rssMB 11260.4765625* proc.memory.availableMB38516.36328125 network.sent 4.085518e+06 disk.drbd1584.in0 disk.sda2.in0 disk.drbd1817.out0 disk.drbd1285.out0 disk.drbd1356.in0 disk.drbd1931.out0 disk.sda.out 55.49609375 disk.drbd2118.in0 disk.rbd1.out 8.033203125 disk.drbd2111.in0 disk.drbd1938.out0" disk.nvme0n1.out 73.1611328125 disk.drbd1828.out0 disk.drbd1550.in0 disk.drbd1285.in0 disk.drbd1584.out0 disk.drbd2041.in0 disk.drbd1828.in0 disk.sda1.in0 disk.drbd2111.out0 disk.drbd1289.in0 disk.drbd1510.in0 disk.drbd1226.out0) disk./.usagePercent32.34437085578444 disk.rbd1.in0 disk.drbd1673.out0 disk.drbd1931.in0 disk.drbd1552.in0 disk.sda.in0 disk.dm-0.out 73.1611328125 disk.nvme0n1.in 2.47265625 disk.dm-0.in 2.47265625xzJ2 - _wandbruntime2031  _runtime2031Rs  % _timestamp1.7857973183697817e+09 _runtime2031.192746377 _step360 ' lr-AdamW/pg10.00011723767632165744 ' lr-AdamW/pg20.00011723767632165744 trainer/global_step45490_  ' lr-AdamW/pg10.00011723767632165744 ' lr-AdamW/pg20.00011723767632165744 trainer/global_step4549 % _timestamp1.7857973183697817e+09 _runtime2031.192746377 _step360<췧2 - _wandbruntime2031  _runtime2031"t  O NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated78113314366464 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated17915747 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated48054487 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed48044831 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated30942847 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed82409664114688 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed78113297327104 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated30230229 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed48044831 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed76172328733472 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed17915745 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current280 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated80165878631604 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 ?9DeviceStatsMonitor.on_train_bv49atch_start/num_alloc_retries0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated48054487 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _step361 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated82116646471680 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated78284716 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed82116542002688 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated82409825984000 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed30230227 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4003332105216 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 % _timestamp1.7857973183707194e+09 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed78113297327104 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated82116646471680 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed78275058 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 4003244675584 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136<' O F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated48054487 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed48044831 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated30942847 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed82409664114688 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed78113297327104 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated30230229 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed48044831 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed76172328733472 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed17915745 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current280 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated80165878631604 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated48054487 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _step361 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated82116646471680 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated78284716 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed82116542002688 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated82409825984000 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed30230227 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4003332105216 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 % _timestamp1.7857973183707194e+09 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed78113297327104 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated82116646471680 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed78275058 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 4003244675584 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated78113314366464 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated17915747 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888-ݽ2 - _wandbruntime2031  _runtime2031&M& M E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed30236870 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 4004124295168 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed78315412891648 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed17919682 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated78301896 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed82427778539008 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated78301896 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated78315437795328 $ _timestamp1.785797318732943e+09 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed78130464090112 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated30949676 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed78292224 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4004211731968 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated13029992 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated80183496277258 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 trainer/global_step4549 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed80183393434662 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 4004211731968 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 3994324528870 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated82427932012544 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed48055354 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated82134692861440 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed82134588385280 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated30236872 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed30949422 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4112494217216 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated30236872 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated48065024 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed48055354 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed76189068905792 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated78130481129472 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed78130464090112 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed78292224 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed30236870 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current252 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated76189085945152 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed30949422 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4112494217216 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated30236872 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated48065024 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed48055354 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed76189068905792 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated78130481129472 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed78130464090112 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed78292224 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed30236870 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current252 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated76189085945152 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated78301896 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated78315437795328 $ _timestamp1.785797318732943e+09 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed78130464090112 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated30949676 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed78292224 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4004211731968 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated13029992 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated80183496277258 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 trainer/global_step4549 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed80183393434662 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 4004211731968 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 3994324528870 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated82427932012544 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed48055354 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated82134692861440 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed82134588385280 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated30236872 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 EDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed83018876288512 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed48572833 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed30562377 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 % _timestamp1.7857973400843785e+09 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated83018980757504 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 4047240811008 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed77009337349472 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 4047328240640 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated48582489 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current265 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 _runtime2052.907655225 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed31282724 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated18112597 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated13170394 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed13170129 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed81046672868628 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed79135210 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed83018876288512 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4047328240640 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed30562377 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed78971635477504 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4047240811008 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated31282991 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 _step365 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated78971652516864 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed18112595 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated79144868 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed78971635477504 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4037335519156 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated48582489 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current267 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated79158586518528 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 trainer/global_step4599 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated83315362297344 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed83315206719488 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated30562379 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated77009354388832 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 4037421322280 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 4156645104640 B' O ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated83315362297344 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed83315206719488 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated30562379 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated77009354388832 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 4037421322280 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 4156645104640 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed83018876288512 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed48572833 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed30562377 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 % _timestamp1.7857973400843785e+09 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated83018980757504 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 4047240811008 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed77009337349472 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 4047328240640 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated48582489 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current265 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 _runtime2052.907655225 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed31282724 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated18112597 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated13170394 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed13170129 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed81046672868628 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed79135210 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed83018876288512 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4047328240640 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed30562377 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed78971635477504 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4047240811008 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated31282991 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 _step365 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated78971652516864 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed18112595 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated79144868 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed78971635477504 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4037335519156 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated48582489 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current267 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated79158586518528 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 trainer/global_step459942 - _wandbruntime2053  _runtime2053T\ M G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated79162048 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated83037027147264 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated81064393356766 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated30569022 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated31289678 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 4048207867392 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 4048120430592 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed79175768731648 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4157684152320 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed13172899 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed48583356 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current247 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4038212992378 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 Vva lCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed78988802240512 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current245 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated30569022 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated48593026 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed79152376 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 4038298795614 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated78988819279872 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated79162048 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated79175793635328 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed30569020 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed81064290514170 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed31289431 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed79152376 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed83333315925504 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed30569020 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated78988819279872 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated13173144 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 4048120430592 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated83037027147264 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed78988802240512 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 trainer/global_step4599 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392gG & M A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4157684152320 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed13172899 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed48583356 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current247 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4038212992378 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed78988802240512 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current245 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated30569022 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated48593026 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed79152376 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 4038298795614 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated78988819279872 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated79162048 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated79175793635328 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed30569020 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed81064290514170 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed31289431 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed79152376 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed83333315925504 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed30569020 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated78988819279872 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated13173144 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 4048120430592 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated83037027147264 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed78988802240512 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 trainer/global_step4599 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated79162048 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated83037027147264 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated81064393356766 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated30569022 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated31289678 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 4048207867392 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 4048120430592 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed79175768731648 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032.3\2 - _wandbruntime2053  _runtime2053קn  2loss/SRqqt2/assignment_loss1.6886388668563308 /loss/FRt2/detection_loss2.4467952805652953 epoch0 0loss/FRt1/assignment_loss3.9815592050552366 0loss/FRt2/assignment_loss3.1538663161428353 0loss/FBt2/assignment_loss0.4492139075365331 /loss/FRt1/detection_loss2.2449499438206355 1loss/SRqqt2/detection_loss1.0941652151778705 /loss/FBt1/detection_loss1.3897535304228466 trainer/global_step4599 % _timestamp1.7857973404580157e+09 2loss/SRqqt1/assignment_loss1.9684188686884367 0loss/SRqqt1/detection_loss2.505874303959728 %loss/total_loss23.29835437196218 _runtime 2053.28085141 _step367 1loss/FBt1/assignment_loss0.20722231940405833 .loss/FBt2/detection_loss2.167896614332373m1  /loss/FBt1/detection_loss1.3897535304228466 trainer/global_step4599 % _timestamp1.7857973404580157e+09 2loss/SRqqt1/assignment_loss1.9684188686884367 0loss/SRqqt1/detection_loss2.505874303959728 %loss/total_loss23.29835437196218 _runtime 2053.28085141 _step367 1loss/FBt1/assignment_loss0.20722231940405833 .loss/FBt2/detection_loss2.167896614332373 2loss/SRqqt2/assignment_loss1.6886388668563308 /loss/FRt2/detection_loss2.4467952805652953 epoch0 0loss/FRt1/assignment_loss3.9815592050552366 0loss/FRt2/assignment_loss3.1538663161428353 0loss/FBt2/assignment_loss0.4492139075365331 /loss/FRt1/detection_loss2.2449499438206355 1loss/SRqqt2/detection_loss1.0941652151778705J (:  ޵U disk.drbd2111.out0 disk.drbd2118.out0 disk.drbd1226.in0 disk.drbd1817.in0 disk.drbd1931.in0 disk.drbd1550.in0 disk.drbd1289.out0 disk.sda.in0 disk.sda1.out0 disk.rbd1.in0 disk.drbd1673.in0 disk.rbd0.in0 disk.drbd1510.out0 cpu13.937352207476518 proc.cpu.threads29 disk.drbd1226.out0 disk.drbd2078.in0 disk.rbd0.out0 network.recv301260 disk.drbd1817.out0 disk.drbd2041.in0 disk.drbd2041.out0 disk.drbd1552.out0 disk.sda1.in0* proc.memory.availableMB38516.46484375 disk.dm-0.out 74.939453125 disk.drbd1584.out0 disk.drbd2078.out0 disk.drbd1673.out0 disk.drbd1510.in0 disk.sda.out56.4375 disk.drbd1356.out0 disk.rbd1.out 8.189453125 disk.drbd1828.in0 disk.sda2.out56.4375 disk.drbd1828.out0 disk.drbd1938.out0# proc.memory.rssMB 11260.4765625! disk.nvme0n1.out 74.939453125$ memory_percent41.22853875160217) proc.memory.percent17.18212366104126$ disk./.usageGB578.3406982421875 disk.dm-0.in 2.47265625 disk.nvme0n1.in 2.47265625 disk.drbd1965.out0 disk.drbd1550.out0 network.sent 4.19917e+06 disk.drbd2118.in0 disk.drbd1285.in0) disk./.usagePercent32.34383943122288 disk.sda2.in0 disk.drbd1965.in0 disk.drbd1931.out0 disk.drbd1289.in0 disk.drbd1584.in0 disk.drbd1356.in0 disk.drbd1552.in0 disk.drbd1938.in0 disk.drbd1285.out0 disk.drbd2111.in0М (: ޵뗋V gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp41.0 gpu.0.powerWatts96.407' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent32.135666666666665 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0Ԑ (: ѮU gpu.0.gpu27.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp41.0 gpu.0.powerWatts123.747' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent41.249 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0U&X (:  U proc.cpu.threads29 network.recv304731 disk.drbd1552.out0 disk.drbd1828.in0 network.sent 4.247229e+06 disk.drbd2078.in0 disk.drbd1817.out0 disk.drbd1584.in0 disk.drbd1289.in0 disk.drbd1584.out0 disk.nvme0n1.in 2.47265625 disk.drbd2111.out0 disk.sda1.out0 disk.drbd2041.out0 disk.drbd1285.out0 disk.drbd2078.out0 disk.rbd0.in0 disk.drbd1226.in0 disk.drbd2118.in0) proc.memory.percent17.18212366104126 disk.sda2.in0 disk.drbd1673.in0 disk.drbd1931.out0 disk.sda.out 56.86328125) disk./.usagePercent32.34384092458777 disk.drbd1285.in0# disk.nvme0n1.out75.23193359375 disk.drbd1356.in0 disk.rbd1.in0 disk.sda.in0 disk.drbd1817.in0 disk.drbd1938.in0 cpu13.934395103135614 disk.dm-0.in 2.47265625 disk.sda2.out 56.86328125 disk.rbd1.out 8.275390625 disk.drbd1828.out0 disk.drbd1938.out0 disk.sda1.in0# proc.memory.rssMB 11260.4765625 disk.drbd1226.out0 disk.drbd2118.out0 disk.drbd2111.in0 disk.drbd1931.in0 disk.drbd1510.in0 disk.dm-0.out75.23193359375 disk.drbd1965.in0 disk.drbd1965.out0 disk.drbd2041.in0 disk.drbd1552.in0* proc.memory.availableMB38515.62890625 disk.drbd1510.out0 disk.drbd1356.out0 disk.drbd1289.out0 disk.drbd1550.in0$ disk./.usageGB578.3407249450684 disk.drbd1673.out0 disk.drbd1550.out0 disk.rbd0.out0% memory_percent41.229814291000366y2 - _wandbruntime2074  _runtime2074TG"|  ' lr-AdamW/pg10.00011981489497018804 ' lr-AdamW/pg20.00011981489497018804 trainer/global_step4649 % _timestamp1.7857973620834079e+09 _runtime2074.906362254 _step368+  _runtime2074.906362254 _step368 ' lr-AdamW/pg10.00011981489497018804 ' lr-AdamW/pg20.00011981489497018804 trainer/global_step4649 % _timestamp1.7857973620834079e+09$xЗ2 - _wandbruntime2074  _runtime2074eD*' O _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 4201825926656 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed83921210574336 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated83921315043328 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated83921315043328 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated84220903153664 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated30894529 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated79829990667264 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed81927569948136 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed77846345965472 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 4081309785788 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed83921210574336 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated49110491 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed84220743381504 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4201960795136 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated31622478 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated18309447 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 _runtime2074.907558111 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed49100835 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated77846363004832 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4081223982664 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 % _timestamp1.7857973620843532e+09 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated13313031 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated80005020 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current242 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 4091236946432 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed18309445 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed31622236 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed49100835 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 4091324376064 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed30894527 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed79829973627904 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated30894529 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated80018942358528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated79829990667264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 _step369 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated49110491 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed80018917454848 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680N O PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed49100835 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pѪ`('ool.allocated77846363004832 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4081223982664 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 % _timestamp1.7857973620843532e+09 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated13313031 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated80005020 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current242 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 4091236946432 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed18309445 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed31622236 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed49100835 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 4091324376064 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed30894527 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed79829973627904 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated30894529 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated80018942358528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated79829990667264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 _step369 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated49110491 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed80018917454848 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 4201825926656 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed83921210574336 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated83921315043328 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated83921315043328 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated84220903153664 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated30894529 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated79829990667264 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed81927569948136 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed77846345965472 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 4081309785788 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed83921210574336 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated49110491 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed84220743381504 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4201960795136 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated31622478 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated18309447 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 _runtime2074.907558111)2 - _wandbruntime2075  _runtime2075tz& M C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 trainer/global_step4649 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated80022200 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed30901170 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed31628890 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 4092116566016 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated79847157430272 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated18313384 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed84238866432000 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed79847140390912 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated30901172 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated30901172 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current267 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed30901170 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated79847157430272 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed13315508 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated84239024099840 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4092204002816 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed80012528 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 % _timestamp1.7857973624539564e+09 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 4202741860352 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated77863103177152 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated81945290436274 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated80022200 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated49121028 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 4082187259122 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed77863086137792 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed18313382 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4202874624512 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4082101455886 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 _step370 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed49111358 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated83939361433088 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated31629159 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed80036124571648 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed80012528 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated49121028 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated80036149475328 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed49111358 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 _runtime2075.277277613 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated13315775 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 4092116566016 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392_@& M C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 % _timestamp1.7857973624539564e+09 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 4202741860352 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated77863103177152 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated81945290436274 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated80022200 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated49121028 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 4082187259122 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed77863086137792 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed18313382 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4202874624512 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4082101455886 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 _step370 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed49111358 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated83939361433088 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated31629159 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed80036124571648 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed80012528 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated49121028 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated80036149475328 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed49111358 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 _runtime2075.277277613 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated13315775 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 4092116566016 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 trainer/global_step4649 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated80022200 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed30901170 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed31628890 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 4092116566016 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated79847157430272 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated18313384 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed84238866432000 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed79847140390912 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated30901172 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated30901172 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current267 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed30901170 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated79847157430272 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed13315508 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated84239024099840 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4092204002816 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed800125282 - _wandbruntime2075  _runtime2075eKM  0loss/FRt1/assignment_loss3.9155612357875755 2loss/SRqqt2/assignment_loss1.6464059493939083 /loss/FRt1/detection_loss2.2604530477210094 1loss/SRqqt1/detection_loss3.4311095907499918 .loss/FBt1/detection_loss1.933044428359358 /loss/FBt2/detection_loss1.1759675741195683 _runtime2075.278390803 _step371 1loss/SRqqt1/assignment_loss1.631769288527338 % _timestamp1.7857973624555635e+09 1loss/FBt1/assignment_loss0.04006061377370262 1loss/FBt2/assignment_loss0.45547629090578995 trainer/global_step4649 0loss/FRt2/assignment_loss3.1127859331094303 /loss/FRt2/detection_loss2.6513370971840162 1loss/SRqqt2/detection_loss1.1671855771914124 &loss/total_loss23.421156626823098 epoch0M   1loss/SRqqt1/assignment_loss1.631769288527338 % _timestamp1.7857973624555635e+09 1loss/FBt1/assignment_loss0.04006061377370262 1loss/FBt2/assignment_loss0.45547629090578995 trainer/global_step4649 0loss/FRt2/assignment_loss3.1127859331094303 /loss/FRt2/detection_loss2.6513370971840162 1loss/SRqqt2/detection_loss1.1671855771914124 &loss/total_loss23.421156626823098 epoch0 0loss/FRt1/assignment_loss3.9155612357875755 2loss/SRqqt2/assignment_loss1.6464059493939083 /loss/FRt1/detection_loss2.2604530477210094 1loss/SRqqt1/detection_loss3.4311095907499918 .loss/FBt1/detection_loss1.933044428359358 /loss/FBt2/detection_loss1.1759675741195683 _runtime2075.278390803 _step371nK (:  U disk.drbd1550.in0 disk.sda.out 57.1640625$ memory_percent41.22847318649292 proc.cpu.threads29 network.recv305706 disk.drbd1552.out0 disk.nvme0n1.in 2.47265625 disk.rbd1.out 8.337890625 disk.drbd1931.out0 disk.drbd2041.out0 disk.drbd1226.in0 disk.drbd2078.in0 network.sent 4.248235e+06 disk.dm-0.out 77.31640625 disk.sda.in0 disk.sda1.out0 disk.drbd1817.in0 disk.drbd1289.in0 disk.drbd1938.out0 disk.drbd1285.in0$ disk./.usageGB578.3451538085938 disk.drbd1931.in0 disk.drbd1289.out0 disk.rbd0.in0 disk.drbd1584.out0 disk.drbd1817.out0 disk.drbd1552.in0 disk.drbd2118.out0 disk.drbd2111.out0' disk./.usagePercent32.344088609821 disk.drbd2041.in0 disk.drbd1828.out0) proc.memory.availableMB 38516.5078125 cpu13.932357195678755 disk.drbd1673.out0 disk.drbd2078.out0 disk.drbd1510.in0 disk.drbd1673.in0 disk.drbd1285.out0 disk.drbd1356.in0 disk.drbd1356.out0 disk.drbd1510.out0 disk.dm-0.in 2.47265625 di;;sk.drbd1828.in0* proc.memory.percent17.181861400604248 disk.sda2.in0 disk.rbd1.in0 disk.drbd1550.out0 disk.drbd2118.in0 disk.sda2.out 57.1640625 disk.drbd2111.in0 disk.drbd1965.in0 disk.drbd1965.out0 disk.sda1.in0# proc.memory.rssMB 11260.3046875 disk.drbd1226.out0 disk.nvme0n1.out 77.31640625 disk.drbd1938.in0 disk.drbd1584.in0 disk.rbd0.out0/dZ (: V gpu.0.gpu24.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp41.0 gpu.0.powerWatts132.308' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent44.102666666666664 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0_B2 - _wandbruntime2097  _runtime2097a  trainer/global_step4699 $ _timestamp1.785797384350831e+09 _runtime2097.173798307 _step372 ' lr-AdamW/pg10.00012110350429445335 ' lr-AdamW/pg20.00012110350429445335&  ' lr-AdamW/pg10.00012110350429445335 ' lr-AdamW/pg20.00012110350429445335 trainer/global_step4699 $ _timestamp1.785797384350831e+09 _runtime2097.173798307 _step3722 - _wandbruntime2097  _runtime2097YY' O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4135320511488 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4247190964736 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed49628837 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated80879298198528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 _step373 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated31226679 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed85126327293952 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed80855514 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed82808467027644 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current262 trainer/global_step4699 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated49638493 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated80865172 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 4247053999104 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated80688328817664 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated13456193 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed13455933 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed49628837 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed31962228 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed78683354581472 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated80688328817664 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed84823544860160 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 4135320511488 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed31226677 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4125112446172 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed80688311778304 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated31226679 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated18506297 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251DӒ' O TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed13455933 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed49628837 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed31962228 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed78683354581472 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated80688328817664 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed84823544860160 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 4135320511488 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed31226677 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4125112446172 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed80688311778304 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated31226679 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated18506297 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4135320511488 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4247190964736 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed49628837 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated80879298198528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 _step373 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated31226679 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed85126327293952 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed80855514 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed82808467027644 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current262 trainer/global_step4699 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated49638493 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated80865172 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 4247053999104 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated80688328817664 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated134561932 - _wandbruntime2097  _runtime2097S}& M EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 _step374 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated31969286 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed49639360 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated84841695718912 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4248092039168 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated31233322 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed80872680 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed31233320 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated49649030 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed31969032 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed80705478541312 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated80882352 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated31233322 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated85144597354496 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ODeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed85144441783808 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4125989919394 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated80882352 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated18510234 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated13459052 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed18510232 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed31233320 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 4247961372160 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 4126075722630 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed80872680 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated80705495580672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current252 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated82826187515782 % _timestamp1.7857973847252302e+09 _runtime2097.548564259 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed80705478541312 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed80896480411648 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated78700111793152 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed84841591242752 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4136200138240 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated49649030 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed78700094753792L/w M REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated31233322 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated85144597354496 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ODeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_ponw"ol.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed85144441783808 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4125989919394 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated80882352 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated18510234 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated13459052 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed18510232 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed31233320 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 4247961372160 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 4126075722630 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed80872680 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated80705495580672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current252 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated82826187515782 % _timestamp1.7857973847252302e+09 _runtime2097.548564259 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed80705478541312 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed80896480411648 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated78700111793152 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed84841591242752 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4136200138240 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated49649030 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed78700094753792 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 _step374 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated31969286 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed49639360 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated84841695718912 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4248092039168 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated31233322 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed80872680 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed31233320 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated49649030 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed31969032 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed80705478541312 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated80882352/p$2 - _wandbruntime2097  _runtime2097[  .loss/FRt2/detection_loss2.242677094764076 .loss/FBt1/detection_loss1.559444557506189 % _timestamp1.7857973847269633e+09 _runtime2097.549793774 0loss/FRt1/assignment_loss3.9392308787544175 2loss/SRqqt1/assignment_loss1.9493150487542152 1loss/SRqqt2/assignment_loss1.577488082854284 1loss/FBt2/assignment_loss0.49625257096652475 1loss/FBt1/assignment_loss0.09313034162509876 .loss/FRt1/detection_loss2.399393467029723 0loss/SRqqt2/detection_loss0.960528384314643 %loss/total_loss21.70996345901224 epoch0 _step375 0loss/FRt2/assignment_loss2.7143017649650574 1loss/SRqqt1/detection_loss2.3223379279176393 /loss/FBt2/detection_loss1.4558633395603724 trainer/global_step4699g9  0loss/FRt2/assignment_loss2.7143017649650574 1loss/SRqqt1/detection_loss2.3223379279176393 /loss/FBt2/detection_loss1.4558633395603724 trainer/global_step4699 .loss/FRt2/detection_loss2.242677094764076 .loss/FBt1/detection_loss1.559444557506189 % _timestamp1.7857973847269633e+09 _runtime2097.549793774 0loss/FRt1/assignment_loss3.9392308787544175 2loss/SRqqt1/assignment_loss1.9493150487542152 1loss/SRqqt2/assignment_loss1.577488082854284 1loss/FBt2/assignment_loss0.49625257096652475 1loss/FBt1/assignment_loss0.09313034162509876 .loss/FRt1/detection_loss2.399393467029723 0loss/SRqqt2/detection_loss0.960528384314643 %loss/total_loss21.70996345901224 epoch0 _step375_O (:  U disk.dm-0.in 2.47265625 disk.nvme0n1.out 78.0390625 disk.sda2.out 57.64453125 disk.drbd2111.out0 disk.drbd1965.in0 disk.drbd1584.in0 disk.drbd1550.out0# proc.memory.rssMB 11260.3046875 disk.drbd1828.out0 disk.dm-0.out 78.0390625 disk.drbd1584.out0 disk.drbd1550.in0 disk.sda.out 57.64453125 disk.sda1.in0 proc.cpu.threads29 disk.drbd1552.in0 disk.drbd1285.in0 disk.sda1.out0 disk.drbd1931.out0 disk.drbd2041.in0 disk.drbd1552.out0 disk.sda2.in0 disk.drbd1673.out0 disk.drbd1931.in0 disk.drbd1828.in0 disk.drbd1289.in0 disk.drbd1226.in0 disk.drbd2078.in0 disk.rbd1.in0 disk.drbd1289.out0 disk.drbd1938.out0$ memory_percent41.22928977012634 disk.drbd2118.out0 disk.drbd2078.out0 disk.drbd1356.out0 disk.drbd1817.out0 disk.sda.in0 disk.drbd1510.out0 cpu13.930576367956869) disk./.usagePercent32.34431453459445 disk.drbd1817.in0* proc.memory.percent17.181861400604248 disk.nvme0n1.in 2.47265625 disk.drbd2041.out0 disk.drbd1938.in0 disk.rbd0.out0 disk.drbd1285.out0 disk.drbd2118.in0 network.sent 4.292721e+06 network.recv307960 disk.drbd1226.out0 disk.drbd2111.in0 disk.drbd1673.in0 disk.drbd1356.in0 disk.rbd1.out 8.431640625 disk.drbd1965.out0 disk.rbd0.in0* proc.memory.availableMB38515.97265625# disk./.usageGB578.349193572998 disk.drbd1510.in0; (: U gpu.0.gpu34.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp41.0 gpu.0.powerWatts102.304' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent34.101333333333336 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0N (: ͲU gpu.0.gpu18.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp41.0 gpu.0.powerWatts101.209' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent33.736333333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0sR (:  ަV$ disk./.usageGB578.3486442565918 disk.drbd1226.in0 disk.drbd1584.in0# proc.memory.rssMB 11260.3046875 disk.drbd1828.in0 disk.rbd0.in0 disk.dm-0.in 2.47265625 disk.dm-0.out 82.1748046875 disk.drbd2118.out0* disk./.usagePercent32.344283813945374 disk.drbd1965.out0 disk.drbd2041.in0 disk.drbd1289.in0 disk.sda1.in0 disk.sda1.out0) proc.memory.availableMB 38515.4453125 disk.drbd1931.in0 disk.drbd1550.in0 disk.rbd0.out0 disk.drbd1552.in0 disk.drbd1550.out0 disk.sda.in0 disk.drbd2078.in0 disk.nvme0n1.in 2.47265625 disk.rbd1.in0 disk.drbd1673.in0 disk.drbd1931.out0 disk.drbd1938.in0* proc.memory.percent17.181861400604248 disk.sda2.in0 disk.drbd1285.in0 disk.drbd1285.out0 disk.drbd1226.out0 disk.drbd2078.out0 disk.sda2.out 57.921875 cpu13.930818075302696 disk.drbd1289.out0 disk.drbd1510.in0 disk.drbd1510.out0 disk.drbd2118.in0 disk.drbd1356.out0 network.sent 4.339593e+06 disk.drbd1965.in0 disk.drbd1552.out0 disk.drbd1817.in0 disk.sda.out 57.921875$ memory_percent41.23009443283081 disk.drbd2041.out0 disk.drbd1584.out0 disk.drbd1356.in0 network.recv309346 disk.drbd2111.out0 disk.drbd1817.out0 disk.drbd1673.out0 disk.rbd1.out 8.580078125 disk.drbd1828.out0 proc.cpu.threads29 disk.drbd2111.in0 disk.drbd1938.out0" disk.nvme0n1.out 82.1748046875h|2 - _wandbruntime2119  _runtime2119>3  _runtime2119.215417624 _step376 ' lr-AdamW/pg10.00012239211361871866 ' lr-AdamW/pg20.00012239211361871866 trainer/global_step4749 % _timestamp1.7857974063924234e+09,\  _step376 ' lr-AdamW/pg10.00012239211361871866 ' lr-AdamW/pg20.00012239211361871866 trainer/global_step4749 % _timestamp1.7857974063924234e+09 _runtime2119.215417624E2 - _wandbruntime2119  _runtime2119g3U' O _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated79520380236832 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed79520363197472 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated81546666968064 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4179229217280 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated81725324 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated18703147 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed81546649928704 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated85725983614976 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4169000909680 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 4292228061184 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed31558827 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated81739654038528 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current254 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed50156839 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated13599074 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated31558829 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated85725983614976 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed81739629134848 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed50156839 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed85725879145984 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed81546649928704 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed18703145 _runtime 2119.21669981 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated50166495 _step377 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 4179316646912 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated81546666968064 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed13598822 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed32301967 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed85725879145984 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4292365026816 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 4169086712804 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated50166495 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed83689364107152 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 trainer/global_step4749 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed31558827 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4179316646912 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated32302221 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current252 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated83689466949636 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed81715666 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4169000909680 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 4292228061184 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed31558827 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated81739654038528 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current254 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed50156839 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated13599074 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated31558829 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated85725983614976 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed81739629134848 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed50156839 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed85725879145984 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed81546649928704 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed18703145 _runtime 2119.21669981 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated50166495 _step377 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.curd rent17039360 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 4179316646912 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated81546666968064 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed13598822 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed32301967 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed85725879145984 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4292365026816 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 4169086712804 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated50166495 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed83689364107152 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 trainer/global_step4749 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed31558827 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4179316646912 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated32302221 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0+?2 - _wandbruntime2119  _runtime21195m& M XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 4180108836864 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated81756861155328 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current239 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4169878382902 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated50177032 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated18707084 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated85744030004736 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed81563816691712 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed81732832 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed81756836251648 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed79537103369792 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed50167362 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated81742504 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated81742504 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated83707084595290 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed85743925528576 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4180196273664 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed86049966975488 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated50177032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 4169964186138 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated81563833731072 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated31565472 _step378 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed18707082 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 4293130723840 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed31565470 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed32308779 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _runtime2119.589020964 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160aE)& M \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 4169964186138 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated81563833731072 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated31565472 _step378 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed18707082 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 4293130723840 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed31565470 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed32308779 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _runtime2119.589020964 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 4180108836864 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated81756861155328 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current239 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4169878382902 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated50177032 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated18707084 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated85744030004736 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed81563816691712 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed81732832 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed81756836251648 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed79537103369792 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed50167362 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated81742504 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated81742504 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated83707084595290 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed85743925528576 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4180196273664 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed86049966975488 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated50177032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251@S2 - _wandbruntime2119  _runtime21194  2loss/SRqqt1/assignment_loss1.4802992475032806 1loss/FBt1/assignment_loss0.02246426033861874 0loss/FBt2/assignment_loss0.4934409705715047 -loss/FRt1/detection_loss2.42656786885916 1loss/SRqqt1/detection_loss1.8879677846852458 0loss/SRqqt2/detection_loss1.074066745925201 _runtime2119.590148339 /loss/FRt2/assignment_loss2.904783018564774 2loss/SRqqt2/assignment_loss1.6237472622509461 /loss/FBt1/detection_loss1.5144130199833563 %loss/total_loss22.11037540036156 epoch0 trainer/global_step4749 % _timestamp1.7857974067673159e+09 _step379 /loss/FRt1/assignment_loss4.073805612676284 .loss/FRt2/detection_loss2.520116672930071 /loss/FBt2/detection_loss2.0887029360731173a!  .loss/FRt2/detection_loss2.520116672930071 /loss/FBt2/detection_loss2.0887029360731173 2loss/SRqqt1/assignment_loss1.4802992475032806 1loss/FBt1/assignment_loss0.02246426033861874 0loss/FBt2/assignment_loss0.4934409705715047 -loss/FRt1/detection_loss2.42656786885916 1loss/SRqqt1/detection_loss1.8879677846852458 0loss/SRqqt2/detection_loss1.074066745925201 _runtime2119.590148339 /loss/FRt2/assignment_loss2.904783018564774 2loss/SRqqt2/assignment_loss1.6237472622509461 /loss/FBt1/detection_loss1.5144130199833563 %loss/total_loss22.11037540036156 epoch0 trainer/global_step4749 % _timestamp1.7857974067673159e+09 _step379 /loss/FRt1/assignment_loss4.073805612676284! (: έU gpu.0.gpu0.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp41.0 gpu.0.powerWatts96.128' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent32.04266666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0a!%R (:  U proc.cpu.threads29 disk.drbd2111.in0 disk.drbd2041.out0 disk.drbd1938.out0 disk.drbd1931.in0 disk.drbd1931.out0 disk.drbd1552.in0 disk.rbd0.out0 disk.sda1.out0 disk.drbd1226.out0 disk.drbd2111.out0 disk.drbd2118.out0 disk.drbd1226.in0 disk.drbd1584.out0 disk.drbd1356.in0 disk.drbd1552.out0* proc.memory.percent17.181861400604248 disk.drbd1510.in0 disk.sda2.out 58.3046875 disk.drbd2041.in0 disk.drbd1965.in0 disk.rbd0.in0 disk.sda1.in0 disk.drbd1285.in0 disk.dm-0.out 87.5302734375 cpu13.932260127074903 disk.rbd1.in0 disk.drbd1289.in0 disk.sda.out 58.3046875 disk.drbd1510.out0$ memory_percent41.22927188873291# proc.memory.rssMB 11260.3046875 disk.drbd1673.in0 disk.dm-0.in 2.47265625( proc.memory.availableMB 38515.984375 network.recv310700 disk.drbd1550.out0) disk./.usagePercent32.34424434644482 disk.drbd2118.in0 disk.drbd1817.out0 disk.drbd1965.out0 disk.drbd1828.out0 disk.sda.in0 disk.drbd1285.out0 disk.drbd2078.out0 disk.nvme0n1.in 2.47265625 disk.drbd1356.out0 disk.drbd1673.out0$ disk./.usageGB578.3479385375977 disk.drbd1584.in0 disk.sda2.in0 disk.drbd1550.in0 disk.drbd1289.out0 disk.drbd2078.in0" disk.nvme0n1.out 87.5302734375 disk.drbd1817.in0 disk.rbd1.out 8.580078125 disk.drbd1828.in0 disk.drbd1938.in0 network.sent 4.343357e+06XN2 - _wandbruntime2141  _runtime2141IOV  trainer/global_step4799 % _timestamp1.7857974286872914e+09 _runtime2141.510271366 _step380 ' lr-AdamW/pg10.00012368072294298397 ' lr-AdamW/pg20.00012368072294298397t  _step380 ' lr-AdamW/pg10.00012368072294298397 ' lr-AdamW/pg20.00012368072294298397 trainer/global_step4799 % _timestamp1.7857974286872914e+09 _runtime2141.510271366I2 - _wandbruntime2141  _runtime2141!: O SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated82585476 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 4337399334912 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed84570261186660 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 % _timestamp1.7857974286882625e+09 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated82405005118464 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed50684841 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated86937539887616 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated82405005118464 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4223225352704 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed80357371813472 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed86628213431808 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current256 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 trainer/global_step4799 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated86628317900800 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated50694497 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated18899997 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4223312782336 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated13743707 VG tHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated31890979 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed82404988079104 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 4223312782336 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed18899995 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed82599984974848 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated86628317900800 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed82575818 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 4223225352704 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed32643448 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated31890979 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed13743453 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated32643704 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated82600009878528 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed31890977 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current254 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4212889373188 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed86937384309760 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121f`' O TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed84570261186660 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 % _timestamp1.7857974286882625e+09 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated82405005118464 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed50684841 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated86937539887616 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated82405005118464 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4223225352704 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed80357371813472 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed86628213431808 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current256 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 trainer/global_step4799 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated86628317900800 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated50694497 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated18899997 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4223312782336 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated13743707 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated31890979 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed82404988079104 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 4223312782336 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed18899995 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed82599984974848 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated86628317900800 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed82575818 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 4223225352704 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed32643448 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated31890979 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed13743453 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated32643704 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated82600009878528 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed31890977 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current254 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4212889373188 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed86937384309760 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated82585476 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 4337399334912 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0=j2 - _wandbruntime2141  _runtime21418'& M D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed32650216 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4224192409088 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated82602656 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated31897622 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated80374129025152 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed31897620 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed50695364 trainer/global_step4799 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current244 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated86646364290560 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated82422171881472 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed82592984 % _timestamp1.7857974290504427e+09 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated82602656 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 4224192409088 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed80374111985792 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated86646364290560 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed18903932 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated82617216995328 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed84587878832202 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated50705034 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed82422154842112 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed31897620 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4338435163136 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed82592984 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 _step382 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated31897622 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 4224104972288 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated18903934 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated84587981674798 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed50695364 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4213766846410 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed86646259814400 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current242 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 4338298204672 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed32650216 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4224192409088 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated82602656 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated31897622 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated80374129025152 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed31897620 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed50695364 trainer/global_step4799 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current244 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated86646364290560 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated82422171881472 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed82592984 % _timestamp1.7857974290504427e+09 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated82602656 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 4224192409088 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed80374111985792 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated86646364290560 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed18903932 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated82617216995328 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed84587878832202 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated50705034 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed82422154842112 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed31897620 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4338435163136 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed82592984 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 _step382 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated31897622 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 4224104972288 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated18903934 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated84587981674798 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251¹=2 - _wandbruntime2141  _runtime2141?!  .loss/FBt1/detection_loss5.809701086218028 /loss/FBt2/detection_loss2.1305332660675047 %loss/total_loss26.61933709144217 epoch0 /loss/FRt1/assignment_loss3.770709171207673 0loss/FRt2/assignment_loss3.3216168701648714 2loss/SRqqt2/assignment_loss1.4663930368977924 .loss/FRt2/detection_loss2.555536413192749 1loss/SRqqt1/detection_loss2.3374079394464693 trainer/global_step4799 2loss/SRqqt1/assignment_loss1.4171156516919534 2loss/FBt1/assignment_loss0.034472929616472324 1loss/FBt2/assignment_loss0.45860558152198794 % _timestamp1.7857974290522118e+09 _runtime2141.875055465 /loss/FRt1/detection_loss2.3314589076085923 1loss/SRqqt2/detection_loss0.9857862378080744 _step383glj   1loss/SRqqt1/detection_loss2.3374079394464693 trainer/global_step4799 2loss/SRqqt1/assignment_loss1.4171156516919534 2loss/FBt1/assignment_loss0.034472929616472324 1loss/FBt2/assignment_loss0.45860558152198794 % _timestamp1.7857974290522118e+09 _runtime2141.875055465 /loss/FRt1/detection_loss2.3314589076085923 1loss/SRqqt2/detection_loss0.9857862378080744 _step383 .loss/FBt1/detection_loss5.809701086218028 /loss/FBt2/detection_loss2.1305332660675047 %loss/total_loss26.61933709144217 epoch0 /loss/FRt1/assignment_loss3.770709171207673 0loss/FRt2/assignment_loss3.3216168701648714 2loss/SRqqt2/assignment_loss1.4663930368977924 .loss/FRt2/detection_loss2.555536413192749W (:  U# disk.nvme0n1.out87.92529296875 disk.drbd1931.in0 disk.rbd0.in0 disk.drbd1226.in0 disk.drbd1285.in0 disk.drbd1285.out0 disk.nvme0n1.in 2.47265625 network.sent 4.388356e+06 network.recv312910 disk.drbd1510.in0 disk.drbd2078.in0 disk.drbd2111.out0# proc.memory.rssMB 11260.3046875 disk.sda.in0 disk.drbd1552.in0 disk.drbd1938.out0 disk.sda2.in0 disk.sda2.out 58.6796875 disk.drbd1817.in0 disk.drbd1552.out0 disk.rbd0.out0 disk.dm-0.out87.92529296875 cpu13.933458932576203 disk.drbd1965.in0 disk.drbd2041.out0 disk.drbd1828.out0 disk.drbd1289.out0 disk.drbd1817.out0$ disk./.usageGB578.3478507995605 disk.drbd1673.in0 disk.drbd1931.out0 disk.drbd1226.out0 disk.drbd2078.out0 disk.drbd1584.in0 disk.drbd1356.out0 disk.sda1.in0 disk.drbd2041.in0 disk.drbd1550.in0 disk.drbd1938.in0 disk.drbd2118.out0* proc.memory.percent17.181861400604248 disk.drbd1965.out0 disk.dm-0.in 2.47265625 disk.drbd1584.out0 disk.drbd2118.in0 disk.rbd1.in0 disk.sda.out 58.6796875% memory_percent41.228604316711426 disk.drbd1289.in0 disk.drbd1356.in0 disk.drbd2111.in0 disk.drbd1550.out0 disk.drbd1510.out0( proc.memory.availableMB 38516.421875* disk./.usagePercent32.344239439674475 disk.drbd1828.in0 proc.cpu.threads29 disk.rbd1.out 8.6123046875 disk.drbd1673.out0 disk.sda1.out0ԩ (: ΆU gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp41.0 gpu.0.powerWatts123.077' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent41.025666666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0; (: Ƕ됥U gpu.0.gpu37.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts122.112' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent40.70399999999999 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0I^U (:  ǶU disk.drbd1552.in0 proc.cpu.threads29 disk.sda1.out0 disk.drbd1510.in0 disk.sda2.out59.03125 disk.drbd1817.in0 cpu13.934746136905506 disk.sda.in0 disk.drbd2078.in0 disk.drbd1965.in0* disk./.usagePercent32.344238372985274 disk.drbd1285.in0 disk.drbd2118.out0 disk.drbd1673.in0 disk.drbd2041.in0 disk.drbd1550.in0 network.sent 4.433375e+06 disk.dm-0.out88.17919921875 disk.drbd2041.out0* proc.memory.availableMB38515.67578125 disk.drbd1938.in0 disk.drbd1510.out0 disk.nvme0n1.in 2.47265625 disk.drbd1931.out0 network.recv315404 disk.drbd1289.in0 disk.drbd1584.out0 disk.drbd1356.out0 disk.drbd1673.out0 disk.drbd1289.out0 disk.rbd0.in0 disk.drbd2118.in0 disk.drbd1356.in0 disk.drbd1931.in0 disk.drbd1550.out0$ disk./.usageGB578.3478317260742 disk.drbd2078.out0* proc.memory.percent17.181861400604248 disk.drbd1226.in0# disk.nvme0n1.out88.17919921875% memory_percent41.229742765426636 disk.sda.out59.03125 disk.sda1.in0 disk.drbd1285.out0 disk.sda2.in0 disk.drbd1828.out0 disk.rbd0.out0 disk.dm-0.in 2.47265625 disk.drbd2111.out0 disk.drbd1817.out0 disk.drbd1965.out0 disk.rbd1.out 8.7060546875 disk.drbd1828.in0 disk.drbd1584.in0 disk.rbd1.in0 disk.drbd1552.out0# proc.memory.rssMB 11260.3046875 disk.drbd1938.out0 disk.drbd1226.out0 disk.drbd2111.in0$2 - _wandbruntime2163  _runtime2163ԝ^  ' lr-AdamW/pg10.00012496933226724928 ' lr-AdamW/pg20.00012496933226724928 trainer/global_step4849 % _timestamp1.7857974508883026e+09 _runtime 2163.71122315 _step384x  ' lr-AdamW/pg10.00012496933226724928 ' lr-AdamW/pg20.00012496933226724928 trainer/global_step4849 % _timestamp1.7857974508883026e+09 _runtime 2163.71122315 _step384kN~2 - _wandbruntime2163  _runtime2163' O QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 4382582206464 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed83263326229504 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current248 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed13886487 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated32223129 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated13886735 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 trainer/global_step4849 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4382714977792 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed19096845 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current250 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed51212843 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated51222499 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed83263326229504 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated87843080696320 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed85451158266168 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed32983332 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4267221488128 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed87530547717632 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated83263343268864 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated32223129 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed83435970 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated83445628 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed32223127 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 4267221488128 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated83263343268864 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated51222499 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated83460365718528 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated85451261108652 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4267308917760 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated32983582 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated87530652186624 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed87530547717632 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated19096847 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed51212843 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 :' O H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4267221488128 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed87530547717632 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated83263343268864 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated32223129 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed83435970 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated83445628 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed32223127 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 4267221488128 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated83263343268864 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated51222499 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated83460365718528 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated85451261108652 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4267308917760 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated32983582 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated87530652186624 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed87530547717632 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated19096847 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed51212843 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 4382582206464 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed83263326229504 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current248 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed13886487 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated32223129 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated13886735 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 trainer/global_step4849 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4382714977792 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed19096845 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current250 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed51212843 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated51222499 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed83263326229504 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated87843080696320 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed85451158266168 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed32983332 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated83462808 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed19100782 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 4268188544512 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4268188544512 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated32990349 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed13889330 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed83477547931648 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 trainer/global_step4849 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 4268101107712 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed83453136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 _step386 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated51233036 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed87861040001536 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 4383492069888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated85468878754306 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated32229772 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 $ _timestamp1.785797451255017e+09 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 K>DeviceStatsMonitor.on_train_batch_end/inactive_sp+:lit.all.freed32990112 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed87548594100224 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed83453136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4383616445440 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated83477572835328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed51223366 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated83280510031872 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 149279232 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated81211137641152 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated83462808 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 124375552 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated32229772 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated87861189280768 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed85468775911710 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4257655309918 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated87548698576384 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed83280492992512 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed32229770 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 4257741113154 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed32990112 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed87548594100224 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed83453136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4383616445440 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated83477572835328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed51223366 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated83280510031872 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 149279232 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated81211137641152 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated83462808 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 124375552 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated32229772 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated87861189280768 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed85468775911710 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4257655309918 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated87548698576384 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed83280492992512 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed32229770 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 4257741113154 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated83462808 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed19100782 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 4268188544512 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4268188544512 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated32990349 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed13889330 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed83477547931648 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 trainer/global_step4849 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 4268101107712 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed83453136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 _step386 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated51233036 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed87861040001536 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 4383492069888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated85468878754306 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130\V2 - _wandbruntime2164  _runtime216430  1loss/SRqqt1/detection_loss2.2467019994381547 .loss/FBt2/detection_loss3.033642257962908 trainer/global_step4849 0loss/FRt1/assignment_loss3.7230165553092958 0loss/FRt2/assignment_loss2.9863727678512704 1loss/SRqqt1/assignment_loss1.438866023506437 0loss/SRqqt2/detection_loss0.874046142369106 /loss/FBt1/detection_loss1.4206100547172469 % _timestamp1.7857974512561848e+09 _runtime2164.078982928 1loss/SRqqt2/assignment_loss1.464591549840328 /loss/FRt1/detection_loss2.5055114701390266 /loss/FRt2/detection_loss2.4697505659584342 epoch0 _step387 %loss/total_loss22.59458145180298 1loss/FBt1/assignment_loss0.09534172878376475 0loss/FBt2/assignment_loss0.3361303359270096z  0loss/FRt1/assignment_loss3.7230165553092958 0loss/FRt2/assignment_loss2.9863727678512704 1loss/SRqqt1/assignment_loss1.438866023506437 0loss/SRqqt2/detection_loss0.874046142369106 /loss/FBt1/detection_loss1.4206100547172469 % _timestamp1.7857974512561848e+09 _runtime2164.078982928 1loss/SRqqt2/assignment_loss1.464591549840328 /loss/FRt1/detection_loss2.5055114701390266 /loss/FRt2/detection_loss2.4697505659584342 epoch0 _step387 %loss/total_loss22.59458145180298 1loss/FBt1/assignment_loss0.09534172878376475 0loss/FBt2/assignment_loss0.3361303359270096 1loss/SRqqt1/detection_loss2.2467019994381547 .loss/FBt2/detection_loss3.033642257962908 trainer/global_step4849R,EV (:  ֶU disk.drbd2111.in0 disk.drbd1673.out0 disk.drbd1550.in0 disk.sda.out 59.46484375 disk.rbd1.in0 disk.drbd2041.in0 disk.drbd1817.out0 disk.drbd1285.out0 disk.drbd2041.out0 disk.rbd0.out0 disk.sda1.out0 disk.dm-0.in 2.47265625 disk.drbd2118.out0 disk.sda2.out 59.46484375 disk.drbd1552.in0 disk.drbd1938.out0 disk.drbd1510.in0 disk.drbd1817.in0 disk.drbd1931.in0 disk.drbd1289.out0 disk.dm-0.out 88.8701171875 disk.drbd1226.in0 disk.sda2.in0 disk.drbd1828.out0 disk.sda1.in0 disk.drbd1550.out0 disk.drbd1289.in0 disk.drbd1938.in0 disk.drbd2078.out0 disk.nvme0n1.in 2.47265625 disk.drbd1356.out0$ memory_percent41.22864603996277* proc.memory.availableMB38516.39453125 disk.rbd1.out 8.7685546875( disk./.usagePercent32.3442364529447 disk.drbd1584.in0" disk.nvme0n1.out 88.8701171875 disk.drbd1510.out0 disk.drbd1226.out0 cpu13.936856060292655 network.sent 4.433483e+06 disk.drbd2078.in0 disk.drbd2111.out0# proc.memory.rssMB 11260.3046875 disk.drbd1931.out0 disk.drbd1285.in0$ disk./.usageGB578.3477973937988 disk.drbd1584.out0 disk.drbd2118.in0 disk.drbd1828.in0* proc.memory.percent17.181861400604248 network.recv315512 disk.drbd1356.in0 disk.drbd1673.in0 disk.drbd1552.out0 disk.sda.in0 proc.cpu.threads29 disk.drbd1965.in0 disk.drbd1965.out0 disk.rbd0.in0 (: ֶV gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts127.076' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent42.358666666666664 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0^Hq{2 - _wandbruntime2186  _runtime2186}+  ' lr-AdamW/pg20.00012625794159151456 trainer/global_step4899 % _timestamp1.7857974736184077e+09 _runtime2186.441408546 _step388 ' lr-AdamW/pg10.00012625794159151456**  % _timestamp1.7857974736184077e+09 _runtime2186.441408546 _step388 ' lr-AdamW/pg10.00012625794159151456 ' lr-AdamW/pg20.00012625794159151456 trainer/global_step4899ܼ=2 - _wandbruntime2186  _runtime2186'(A' O JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated84121681419264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 trainer/global_step4899 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 4311305053184 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed33323765 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated84320721558528 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated19293697 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed84121664379904 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed88432882003456 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed88748504260608 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 4300752103328 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4311217623552 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4427940377088 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed84121664379904 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 4427807605760 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed32555277 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated32555279 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed14030070 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated82031406084832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated51750501 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated32555279 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated88432986472448 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed84320696654848 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed88432882003456 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4300666300204 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated33324025 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 _runtime2186.442609459 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 _step389 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 4311217623552 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated84305780 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated14030328 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated88432986472448 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated86332158188160 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated88748661935616 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed82031389045472 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed19293695 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current260 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 % _timestamp1.7857974736193974e+09 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed33323765 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated84320721558528 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 IDeviceStaStsMonitor.on_train_batch_start/oversize_segments.peak0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated19293697 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed84121664379904 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed88432882003456 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed88748504260608 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 4300752103328 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4311217623552 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4427940377088 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed84121664379904 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 4427807605760 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed32555277 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated32555279 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed14030070 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated82031406084832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated51750501 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated32555279 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated88432986472448 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed84320696654848 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed88432882003456 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4300666300204 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated33324025 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 _runtime2186.442609459 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 _step389 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 4311217623552 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated84305780 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated14030328 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated88432986472448 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated86332158188160 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated88748661935616 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed82031389045472 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed19293695 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current260 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 % _timestamp1.7857974736193974e+09 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 trainer/global_step4899 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed84313288 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed32561920 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 _step390 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed51751368 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 4312097243136 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed14032940 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed33330572 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated32561922 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed32561920 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed84138831142912 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated84322960 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated84322960 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 % _timestamp1.7857974739942644e+09 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated32561922 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed51751368 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated86349775833814 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated33330825 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current253 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 4301629576662 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed88766614162944 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated88451032862208 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated19297634 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed88450928386048 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed86349672991218 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121ME& M bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated84337928675328 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed84337903771648 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated82048146257152 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4301543773426 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated88451032862208 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 trainer/global_step4899 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed84313288 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed32561920 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 _step390 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed51751368 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 4312097243136 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed14032940 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed33330572 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated32561922 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed32561920 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed84138831142912 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated84322960 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated84322960 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 % _timestamp1.7857974739942644e+09 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated32561922 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed51751368 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated86349775833814 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated33330825 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current253 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 4301629576662 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed88766614162944 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated88451032862208 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated19297634 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed88450928386048 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed86349672991218 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed82048129217792U2 - _wandbruntime2186  _runtime2186  _runtime 2186.81963429 _step391 /loss/FRt1/assignment_loss3.816601096519402 0loss/FRt2/assignment_loss2.9511351602664893 2loss/SRqqt2/assignment_loss1.6949203988381971 1loss/SRqqt1/detection_loss2.5748043235488294 1loss/SRqqt2/detection_loss0.9919576011307072 trainer/global_step4899 0loss/FBt1/assignment_loss0.0244906735859729 0loss/FBt2/assignment_loss0.8815964375583766 &loss/total_loss23.495537199954242 2loss/SRqqt1/assignment_loss1.7288219779729843 .loss/FBt1/detection_loss1.373203129740432 % _timestamp1.7857974739967031e+09 /loss/FRt1/detection_loss2.3238288020449027 /loss/FRt2/detection_loss2.2656692262145057 /loss/FBt2/detection_loss2.8685083725334457 epoch0||  2loss/SRqqt1/assignment_loss1.7288219779729843 .loss/FBt1/detection_loss1.373203129740432 % _timestamp1.7857974739967031e+09 /loss/FRt1/detection_loss2.3238288020449027 /loss/FRt2/detection_loss2.2656692262145057 /loss/FBt2/detection_loss2.8685083725334457 epoch0 _runtime 2186.81963429 _step391 /loss/FRt1/assignment_loss3.816601096519402 0loss/FRt2/assignment_loss2.9511351602664893 2loss/SRqqt2/assignment_loss1.6949203988381971 1loss/SRqqt1/detection_loss2.5748043235488294 1loss/SRqqt2/detection_loss0.9919576011307072 trainer/global_step4899 0loss/FBt1/assignment_loss0.0244906735859729 0loss/FBt2/assignment_loss0.8815964375583766 &loss/total_loss23.495537199954242vS (: U gpu.0.gpu22.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts103.462' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent34.48733333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0η7T (:  U disk.drbd1673.in0 disk.drbd1550.in0 disk.drbd1289.out0 disk.drbd1938.out0 disk.sda.out 60.1015625 disk.drbd1510.out0 disk.dm-0.in 2.47265625 disk.drbd1965.in0 disk.drbd1673.out0 disk.drbd1285.in0" disk.nvme0n1.out 89.0654296875 disk.drbd1584.out0 disk.drbd2118.in0 disk.drbd2111.in0 disk.drbd1817.in0$ memory_percent41.22985601425171$ disk./.usageGB578.3478355407715 disk.dm-0.out 89.0654296875# proc.memory.rssMB 11260.3046875 disk.drbd2111.out0 disk.drbd1938.in0 disk.drbd1226.in0 disk.drbd2078.out0 disk.drbd2041.in0 disk.drbd1552.in0 disk.drbd1828.in0 disk.drbd1356.in0 disk.drbd2041.out0 disk.drbd1510.in0 disk.drbd2118.out0 network.recv318067 disk.sda.in0 disk.drbd1828.out0 disk.sda1.out0 disk.nvme0n1.in 2.47265625 disk.drbd1965.out0 disk.rbd1.in0 disk.drbd1931.in0 cpu13.939216588058425 disk.sda1.in0* proc.memory.percent17.181861400604248 disk.sda2.out 60.1015625 disk.drbd1285.out0 disk.sda2.in0 disk.drbd1226.out0 disk.drbd2078.in0 disk.drbd1817.out0 disk.rbd0.in0 proc.cpu.threads29 disk.drbd1931.out0 disk.drbd1552.out0 disk.drbd1550.out0 disk.drbd1356.out0) proc.memory.availableMB 38515.6015625 network.sent 4.481077e+06 disk.rbd1.out 8.7685546875 disk.drbd1289.in0 disk.rbd0.out0 disk.drbd1584.in0) disk./.usagePercent32.34423858632311 ؛ (: U gpu.0.gpu11.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts125.414' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent41.80466666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0x (:  V disk.drbd1938.in0 disk.rbd0.in0 disk.sda1.out0 disk.dm-0.in 2.47265625 disk.dm-0.out89.30517578125 disk.drbd1673.out0 disk.drbd1226.out0 disk.drbd1356.in0 disk.drbd2111.in0 disk.drbd1817.in0 cpu13.94021888734389 disk.drbd1552.out0 disk.drbd1828.out0 disk.drbd1965.out0 disk.drbd1931.out0 disk.drbd1550.in0 disk.rbd0.out0 disk.sda2.out60.4375$ memory_percent41.22987389564514 disk.drbd2041.in0 disk.drbd1285.out0 disk.drbd1510.out0 disk.drbd1584.in0 disk.drbd1550.out0 disk.drbd1510.in0 disk.drbd2118.in0 disk.drbd2111.out0 disk.rbd1.in0 disk.nvme0n1.in 2.47265625 disk.sda2.in0 network.recv318175 disk.sda.out60.4375 disk.sda1.in0 disk.drbd1817.out0 disk.drbd1931.in0 disk.sda.in0 disk.drbd1552.in0 disk.drbd1285.in0 disk.rbd1.out 8.80078125* proc.memory.percent17.181885242462158# proc.memory.rssMB 11260.3203125 disk.drbd2041.out0 disk.drbd1226.in0# disk.nvme0n1.out89.30517578125 disk.drbd1965.in0$ disk./.usageGB578.3478546142578 disk.drbd2118.out0 disk.drbd1356.out0 proc.cpu.threads29 network.sent 4.481185e+06* disk./.usagePercent32.344239653012316 disk.drbd1289.out0 disk.drbd1938.out0 disk.drbd2078.out0 disk.drbd1828.in0 disk.drbd207Y#}8.in0 disk.drbd1584.out0* proc.memory.availableMB38515.58984375 disk.drbd1673.in0 disk.drbd1289.in0$b2 - _wandbruntime2208  _runtime2208Q?  _step392 ' lr-AdamW/pg10.00012754655091577988 ' lr-AdamW/pg20.00012754655091577988 trainer/global_step4949 % _timestamp1.7857974958003018e+09 _runtime2208.623258784(G  ' lr-AdamW/pg10.00012754655091577988 ' lr-AdamW/pg20.00012754655091577988 trainer/global_step4949 % _timestamp1.7857974958003018e+09 _runtime2208.623258784 _step392׵2 - _wandbruntime2208  _runtime2208~' O L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed32887427 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated85165932 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated32887429 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current253 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 trainer/global_step4949 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated89654168503296 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated14173023 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 4344640566836 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed19490545 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed87212952425184 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 % _timestamp1.7857974958012166e+09 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed52268847 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 4355301188608 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed52268847 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4355301188608 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 4472954139136 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated32887429 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4355213758976 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 _step393 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated84980019569664 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed89335216289280 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed32887427 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated85181077398528 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed89654006633984 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4473091104768 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated89335320758272 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed82868397661472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed89335216289280 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed84980002530304 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed14172770 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed33663315 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated89335320758272 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated19490547 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed85181052494848 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4344554763712 _runtime2208.624513055 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated87213055267668 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated33663570 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated52278503 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated84980019569664 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 4355213758976 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated82868414700832 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed84980002530304 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed89654006633984 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4473091104768 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated89335320758272 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed82868397661472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed89335216289280 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed84980002530304 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed14172770 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed33663315 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated89335320758272 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated19490547 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed85181052494848 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4344554763712 _runtime2208.624513055 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated87213055267668 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated33663570 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated52278503 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated84980019569664 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 4355213758976 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated82868414700832 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed84980002530304 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated85165932 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated32887429 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current253 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 trainer/global_step4949 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated89654168503296 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated14173023 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 4344640566836 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712]W2 - _wandbruntime2208  _runtime2208)& M ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated85183112 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated32894072 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated52289040 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated84997186332672 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed84997169293312 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 _step394 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed32894070 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 4356093378560 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4356180815360 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated85183112 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed52279370 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated87230672913322 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed82885137833792 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated82885154873152 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed19494482 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed85198259611648 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 trainer/global_step4949 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed33670095 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4345432236934 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed84997169293312 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 4473854180352 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated32894072 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed32894070 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed85173440 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated33670361 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed89672113792000 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed52279370 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated19494484 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed87230570070726 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated89353367148032 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 4356180815360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current266 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated14175877 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated85198284515328 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated52289040 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated89672277751296 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed14175613 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130p M R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed87230570070726 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated89353367148032 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 4356180815360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current266 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated14175877 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated85198284515328 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocaY ted52289040 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated89672277751296 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed14175613 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated85183112 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated32894072 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated52289040 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated84997186332672 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed84997169293312 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 _step394 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed32894070 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 4356093378560 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4356180815360 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated85183112 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed52279370 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated87230672913322 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed82885137833792 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated82885154873152 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed19494482 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed85198259611648 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 trainer/global_step4949 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed33670095 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4345432236934 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed84997169293312 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 4473854180352 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated32894072 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed32894070 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed85173440 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated33670361 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed89672113792000 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed52279370 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated19494484 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0@2 - _wandbruntime2208  _runtime2208\i  0loss/FBt2/assignment_loss0.3040199548005538 .loss/FRt2/detection_loss2.334443857578131 /loss/SRqqt1/detection_loss3.18050534974318 0loss/SRqqt2/detection_loss1.078594677335357 /loss/FBt1/detection_loss0.9925631102277975 % _timestamp1.7857974961753268e+09 2loss/FBt1/assignment_loss0.006068845495443042 .loss/FBt2/detection_loss5.210197908537729 /loss/FRt1/assignment_loss3.874970466859879 2loss/SRqqt2/assignment_loss1.4900205399586548 .loss/FRt1/detection_loss2.526755852084006 %loss/total_loss25.31836210902766 epoch0 trainer/global_step4949 /loss/FRt2/assignment_loss2.700156264121716 2loss/SRqqt1/assignment_loss1.6200652822852135 _runtime2208.998161772 _step395ߓ  /loss/FRt1/assignment_loss3.874970466859879 2loss/SRqqt2/assignment_loss1.4900205399586548 .loss/FRt1/detection_loss2.526755852084006 %loss/total_loss25.31836210902766 epoch0 trainer/global_step4949 /loss/FRt2/assignment_loss2.700156264121716 2loss/SRqqt1/assignment_loss1.6200652822852135 _runtime2208.998161772 _step395 0loss/FBt2/assignment_loss0.3040199548005538 .loss/FRt2/detection_loss2.334443857578131 /loss/SRqqt1/detection_loss3.18050534974318 0loss/SRqqt2/detection_loss1.078594677335357 /loss/FBt1/detection_loss0.9925631102277975 % _timestamp1.7857974961753268e+09 2loss/FBt1/assignment_loss0.006068845495443042 .loss/FBt2/detection_loss5.210197908537729D b (: ѝU gpu.0.gpu41.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts120.572' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent40.19066666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0OT (:  U disk.drbd2041.in0 disk.drbd1552.out0 disk.drbd1584.out0* proc.memory.percent17.181885242462158 cpu13.942527238567765 disk.rbd0.out0) disk./.usagePercent32.34423858632311 disk.drbd2111.in0 disk.rbd1.in0 disk.drbd2118.out0 disk.sda2.in0 disk.drbd1817.out0 disk.drbd1965.out0 disk.drbd1673.out0 disk.drbd1931.in0 disk.drbd1828.out0 disk.drbd1938.in0 disk.drbd2041.out0 disk.drbd1550.out0 disk.drbd1285.in0 disk.drbd2078.out0 disk.nvme0n1.in 2.47265625# disk.nvme0n1.out89.58251953125$ memory_percent41.22997522354126) proc.memory.availableMB 38515.5234375 disk.sda2.out 60.8984375 disk.drbd1510.out0 disk.drbd1552.in0 disk.drbd1550.in0 disk.drbd1289.out0 disk.drbd1356.out0# proc.memory.rssMB 11260.3203125 disk.drbd1584.in0$ disk./.usageGB578.3478355407715 disk.drbd1828.in0 disk.drbd1938.out0 disk.sda.out 60.8984375 disk.drbd1226.in0 network.sent 4.528844e+06 disk.drbd2111.out0 disk.rbd1.out 8.98046875 network.recv319976 disk.drbd1817.in0 disk.drbd1289.in0 disk.sda1.out0 disk.drbd1226.out0 proc.cpu.threads29 disk.drbd1965.in0 disk.sda1.in0 disk.drbd2118.in0 disk.drbd1931.out0 disk.sda.in0 disk.dm-0.in 2.47265625 disk.dm-0.out89.58251953125 disk.drbd1356.in0 disk.drbd1673.in0 disk.rbd0.in0 disk.drbd1285.out0 disk.drbd1510.in0 disk.drbd2078.in0ܸ2 - _wandbruntime2231  _runtime2231v  & lr-AdamW/pg20.0001288351602400452 trainer/global_step4999 $ _timestamp1.785797518363331e+09 _runtime2231.186316615 _step396 & lr-AdamW/pg10.0001288351602400452^  & lr-AdamW/pg10.0001288351602400452 & lr-AdamW/pg20.0001288351602400452 trainer/global_step4999 $ _timestamp1.785797518363331e+09 _runtime2231.186316615 _step396w(2 - _wandbruntime2231  _runtime2231 ٽ' O WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated33219579 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated34003312 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated85838357720064 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 4399297324032 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated88093952347176 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed86016426 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed33219577 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated85838357720064 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed90237550575104 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed19687395 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed14315680 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated83705423316832 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated90237655044096 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed85838340680704 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed90237550575104 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4399209894400 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed86041408334848 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated33219579 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated52806505 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 _runtime 2231.18752781 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4388443227220 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 4399209894400 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated86041433238528 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4518253621248 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 _step397 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated90237655044096 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed85838340680704 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current237 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated90559686859776 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 4518122947072 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed83705406277472 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed33219577 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4399297324032 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed34003075 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated52806505 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed52796849 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated19687397 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed88093849504692 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated14315915 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current235 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4399297324032 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed34003075 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated52806505 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed52796849 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated19687397 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed88093849504692 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated14315915 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current235 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed90237550575104 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4399209894400 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed86041408334848 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated33219579 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated52806505 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 _runtime 2231.18752781 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4388443227220 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 4399209894400 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated86041433238528 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4518253621248 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 _step397 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated90237655044096 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed85838340680704 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current237Iv -  qf+_wandbruntime2231  _runtime2231N r& M \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 4389406503678 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed14318501 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated85855524483072 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed83722146449792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4519158357504 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated33226222 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated83722163489152 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated90255701433856 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated90577798712832 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _step398 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated90255701433856 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed86058615451648 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed33226220 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current250 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed90255596957696 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4400176950784 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed88111467150234 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed34009833 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 163959296 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed86033592 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed33226220 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed52807372 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 4400089513984 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated86043264 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated88111569992830 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated52817042 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated86058640355328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4389320700442 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 trainer/global_step4999 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4389320700442 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 trainer/global_step4999 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed14318501 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated85855524483072 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed83722146449792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4519158357504 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated33226222 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated83722163489152 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated90255701433856 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated90577798712832 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _step398 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated90255701433856 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed86058615451648 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed33226220 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current250 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed90255596957696 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4400176950784 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed88111467150234 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed34009833 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 163959296 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed86033592 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed33226220 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed52807372 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 4400089513984 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated86043264 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated88111569992830 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated52817042 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated86058640355328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 ODeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed91139884860928 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed86901764174848 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed86876578 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 trainer/global_step5049 % _timestamp1.7857975404121652e+09 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed19884245 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed84542414893472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 _step401 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4443206029824 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed91139884860928 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ' O L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed53324851 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 4443293459456 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated91139989329920 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated53334507 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated14459391 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated91465222797824 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed91139884860928 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 _runtime2253.235500978 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated86696695870464 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated33551729 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed86696678831104 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated34343638 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed14459135 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed91465060928512 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed53324851 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4432331690728 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed88974746584200 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated91139989329920 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current258 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated86696695870464 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current256 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated33551729 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 4443206029824 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed33551727 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4443293459456 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed34343380 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed91139884860928 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed86901764174848 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed86876578 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 trainer/global_step5049 % _timestamp1.7857975404121652e+09 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed19884245 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed84542414893472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 _step401 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4443206029824 IDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed34350195 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 % _timestamp1.7857975407759352e+09 _step402 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated86903416 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated53345044 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed14462013 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 4564204012032 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed86713845594112 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed86893744 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed88992364229742 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4564332581888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _runtime2253.599297804 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed33558370 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed84559155065792 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4444173086208 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed53335374 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 4444173086208 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated91158035719680 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated34350432 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4433209163950 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated91483328777216 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated86918996195328 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated53345044 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed19888182 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed86893744 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed53335374 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated14462248 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated86713862633472 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated86903416 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed33558370 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated91158035719680 trainer/global_step5049 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated33558372 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed86713845594112 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated33558372 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current235 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated19888184 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251eYx76# M REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed19888182 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed86893744 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed53335374 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated14462248 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated86713862633472 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated86903416 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed33558370 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated91158035719680 trainer/global_step5049 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated33558372 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed86713845594112 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated33558372 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current235 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated19888184 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed34350195 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 % _timestamp1.7857975407759352e+09 _step402 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated86903416 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated53345044 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed14462013 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 4564204012032 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed86713845594112 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed86893744 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed88992364229742 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4564332581888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _runtime2253.599297804 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed33558370 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed84559155065792 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4444173086208 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed53335374 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 4444173086208 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated91158035719680 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated34350432 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4433209163950 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 JDDeviceStau,tsMonitor.on_train_batch_end/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated91483328777216 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated86918996195328 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated53345044Z2 - _wandbruntime2253  _runtime22533  2loss/SRqqt2/assignment_loss1.5278391514174239 1loss/SRqqt2/detection_loss1.0595423415481686 .loss/FBt2/detection_loss2.091218948364258 _runtime2253.600548822 _step403 0loss/FRt2/assignment_loss2.8644731601079303 1loss/FBt1/assignment_loss0.12049320352967438 /loss/FBt2/assignment_loss0.619786141984514 0loss/SRqqt1/detection_loss2.607380375552636 /loss/FBt1/detection_loss1.8923484943973432 " _timestamp1.7857975407777e+09 &loss/total_loss23.896032012340918 .loss/FRt1/detection_loss2.467975800856948 .loss/FRt2/detection_loss2.610095082471768 epoch0 trainer/global_step5049 .loss/FRt1/assignment_loss4.17746817568938 2loss/SRqqt1/assignment_loss1.8574111364208734 [ .loss/FRt1/detection_loss2.467975800856948 .loss/FRt2/detection_loss2.610095082471768 epoch0 trainer/global_step5049 .loss/FRt1/assignment_loss4.17746817568938 2loss/SRqqt1/assignment_loss1.8574111364208734 2loss/SRqqt2/assignment_loss1.5278391514174239 1loss/SRqqt2/detection_loss1.0595423415481686 .loss/FBt2/detection_loss2.091218948364258 _runtime2253.600548822 _step403 0loss/FRt2/assignment_loss2.8644731601079303 1loss/FBt1/assignment_loss0.12049320352967438 /loss/FBt2/assignment_loss0.619786141984514 0loss/SRqqt1/detection_loss2.607380375552636 /loss/FBt1/detection_loss1.8923484943973432 " _timestamp1.7857975407777e+09 &loss/total_loss23.896032012340918~K(: U gpu.0.gpu0.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts97.687' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent32.562333333333335 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0tUR(:  U disk.drbd1931.out0 disk.drbd1552.in0 disk.drbd1510.in0 disk.drbd2078.out0 disk.rbd0.out0 disk.dm-0.in 2.47265625 disk.drbd2118.in0 disk.drbd1673.out0 disk.drbd1285.in0 disk.drbd1226.in0 disk.drbd1584.in0 disk.sda2.in0 network.recv325012 disk.drbd1931.in0 disk.drbd1356.out0 disk.drbd2111.out0 network.sent 4.663611e+06 disk.drbd1550.out0 disk.sda.out 61.890625 disk.nvme0n1.in 2.47265625 disk.drbd1938.out0$ disk./.usageGB578.3397941589355 disk.drbd1673.in0 disk.drbd1828.out0 disk.drbd1285.out0 disk.drbd1510.out0 disk.drbd2078.in0 disk.rbd0.in0 disk.drbd1965.out0 disk.sda1.out0 disk.drbd1584.out0! proc.memory.rssMB 11260.40625 proc.cpu.threads29 disk.drbd1817.in0 disk.rbd1.out 9.0751953125 disk.drbd2118.out0 disk.drbd1938.in0 disk.drbd1817.out0 disk.sda.in0 disk.drbd2111.in0% memory_percent41.229867935180664 disk.rbd1.in0 disk.drbd2041.in0 disk.drbd2041.out0 disk.drbd1552.out0 disk.drbd1550.in0 disk.sda1.in0 disk.drbd1289.in0 disk.dm-0.out91.43505859375 cpu13.945887904648536 disk.drbd1289.out0 disk.drbd1226.out0 disk.drbd1356.in0* disk./.usagePercent32.343788870154604 disk.drbd1965.in0 disk.drbd1828.in0# disk.nvme0n1.out91.43505859375 disk.sda2.out 61.890625* proc.memory.percent17.182016372680664' proc.memory.availableMB 38515.59375'O2- _wandbruntime2274  _runtime2274 trainer/global_step5099 % _timestamp1.7857975621449869e+09 _runtime2274.967962095 _step404 & lr-AdamW/pg10.0001314123788885758 & lr-AdamW/pg20.0001314123788885758 & lr-AdamW/pg10.0001314123788885758 & lr-AdamW/pg20.0001314123788885758 trainer/global_step5099 % _timestamp1.7857975621449869e+09 _runtime2274.967962095 _step404Fi2- _wandbruntime2274  _runtime2274Y 'O ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4476220154236 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current259 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed92370641601536 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current257 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated87555034020864 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated53862509 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed53852853 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated33883879 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated34683805 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed34683546 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated89855746506192 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated92370805568000 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed92042219146752 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated87746388 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated33883879 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated53862509 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed87736730 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 4476305957360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 trainer/global_step5099 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 _runtime 2274.96919659 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed87555016981504 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4608660649472 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0f)'O TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated33883879 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated34683805 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed34683546 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated89855746506192 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated92370805568000 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed92042219146752 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated87746388 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated33883879 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated53862509 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed87736730 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 4476305957360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 trainer/global_step5099 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 _runtime 2274.96919659 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed87555016981504 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4608660649472 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4476220154236 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current259 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed92370641601536 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current257 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated87555034020864 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated53862509 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed53852853 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0s 2- _wandbruntime2275  _runtime2275p3BM G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated87763568 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated85396180721152 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated92060370005504 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated87572200783872 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated34690567 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated33890522 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed85396163681792 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4477097627458 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed33890520 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed92060265529344 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated92060370005504 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 4477183430694 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed53863376 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _runtime2275.339283866 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated87779352035328 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed34690313 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 168153600 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4609562125312 % _timestamp1.7857975625159664e+09 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed87572183744512 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated33890522 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed20085032 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated14605533 ODeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 4488169221632 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 4488081784832 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated87763568 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed87753896 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed87753896 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 143249920 [S =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 trainer/global_step5099 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed34690313 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 168153600 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4609562125312 % _timestamp1.7857975625159664e+09 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed87572183744512 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated33890522 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed20085032 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated14605533 ODeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 4488169221632 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 4488081784832 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated87763568 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed87753896 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed87753896 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 143249920 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 trainer/global_step5099 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated85396180721152 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated92060370005504 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated87572200783872 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated34690567 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated33890522 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed85396163681792 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4477097627458 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed338905202- _wandbruntime2275  _runtime2275n &loss/total_loss20.444582545091542 _step407 /loss/FRt1/detection_loss2.3582076118105935 .loss/FRt2/detection_loss2.348910876040189 1loss/SRqqt2/detection_loss0.8836297608319331 epoch0 % _timestamp1.7857975625176024e+09 _runtime2275.340418926 /loss/FRt2/assignment_loss2.684000742885302 2loss/SRqqt2/assignment_loss1.5338764295659282 trainer/global_step5099 0loss/FRt1/assignment_loss4.0479037421090265 2loss/SRqqt1/assignment_loss1.2530102796024747 1loss/FBt1/assignment_loss0.09231755649521159 -loss/SRqqt1/detection_loss2.468558513042 .loss/FBt1/detection_loss1.679256570244371 0loss/FBt2/detection_loss0.42693081829282975 0loss/FBt2/assignment_loss0.6679796441716866N 0loss/FRt1/assignment_loss4.0479037421090265 2loss/SRqqt1/assignment_loss1.2530102796024747 1loss/FBt1/assignment_loss0.09231755649521159 -loss/SRqqt1/detection_loss2.468558513042 .loss/FBt1/detection_loss1.679256570244371 0loss/FBt2/detection_loss0.42693081829282975 0loss/FBt2/assignment_loss0.6679796441716866 &loss/total_loss20.444582545091542 _step407 /loss/FRt1/detection_loss2.3582076118105935 .loss/FRt2/detection_loss2.348910876040189 1loss/SRqqt2/detection_loss0.8836297608319331 epoch0 % _timestamp1.7857975625176024e+09 _runtime2275.340418926 /loss/FRt2/assignment_loss2.684000742885302 2loss/SRqqt2/assignment_loss1.5338764295659282 trainer/global_step5099홥C(: U gpu.0.gpu0.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts97.777' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent32.592333333333336 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0kO(:  U network.sent 4.708589e+06 network.recv326929 disk.drbd1584.out0# disk.nvme0n1.out91.62646484375 disk.drbd1817.in0 disk.rbd1.out 9.1689453125 disk.drbd1552.out0 disk.drbd1828.out0$ disk./.usageGB578.3397064208984 disk.drbd1285.out0 disk.drbd1938.out0) proc.memory.availableMB 38515.0546875 disk.rbd0.in0 disk.dm-0.in 2.47265625 disk.drbd1931.in0 disk.drbd1931.out0 disk.drbd1552.in0 disk.drbd1289.in0 disk.sda1.out0 disk.drbd1510.out0 disk.drbd2078.in0 disk.sda2.out62.1875 cpu13.943144718444033 disk.rbd0.out0 disk.sda1.in0 disk.drbd1285.in0 disk.drbd1510.in0 disk.drbd1965.out0 disk.drbd1550.out0 disk.drbd2111.out0 disk.drbd1965.in0 disk.drbd1828.in0 disk.nvme0n1.in 2.47265625 disk.drbd2041.in0 proc.cpu.threads29 disk.sda2.in0 disk.drbd1289.out0) disk./.usagePercent32.34378396338427 disk.drbd1226.in0 disk.drbd1226.out0 disk.drbd1584.in0! proc.memory.rssMB 11260.40625 disk.drbd2041.out0% memory_percent41.230690479278564 disk.drbd1356.in0 disk.drbd1817.out0 disk.sda.out62.1875 disk.drbd2111.in0 disk.drbd1673.out0 disk.drbd1938.in0 disk.sda.in0 disk.drbd1673.in0 disk.drbd2118.in0 disk.drbd2118.out0 disk.drbd1550.in0 disk.dm-0.out91.62646484375 disk.drbd2078.out0 disk.drbd1356.out0 disk.rbd1.in0* proc.memory.percent17.182016372680664|E(: η⁈U gpu.0.gpu41.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts98.086' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent32.69533333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors05nQ(:  ηU disk.drbd1552.in0 disk.drbd1550.in0 proc.cpu.threads29 disk.sda.out 62.60546875 disk.drbd1285.out0 disk.drbd1356.in0 disk.drbd1673.in0* proc.memory.percent17.182016372680664 disk.drbd1828.out0 disk.dm-0.in 2.47265625 disk.drbd2118.in0 disk.drbd2111.out0 disk.rbd1.out 9.2314453125 disk.drbd1965.in0$ disk./.usageGB578.3398017883301 disk.drbd2111.in0 cpu13.940861383415841 disk.dm-0.out 92.626953125) disk./.usagePercent32.34378929683028 disk.drbd1226.in0! disk.nvme0n1.out 92.626953125 disk.drbd1817.in0 disk.drbd1285.in0 disk.drbd1584.in0 disk.drbd1673.out0 disk.drbd1828.in0 disk.drbd1510.out0 disk.sda2.out 62.60546875$ memory_percent41.22955799102783! proc.memory.rssMB 11260.40625 disk.drbd1931.in0 disk.drbd1938.in0 disk.drbd1356.out0 disk.drbd2041.in0 disk.drbd1226.out0 disk.drbd2118.out0 disk.drbd1552.out0 disk.rbd0.in0 disk.sda.in0 disk.drbd2078.in0 disk.rbd1.in0 disk.drbd2041.out0 disk.rbd0.out0 disk.nvme0n1.in 2.47265625 disk.drbd1817.out0( proc.memory.availableMB 38515.796875 network.sent 4.708655e+06 network.recv326995 disk.drbd1510.in0 disk.drbd1584.out0 disk.sda2.in0 disk.drbd1965.out0 disk.sda1.in0 disk.drbd1289.in0 disk.drbd2078.out0 disk.drbd1931.out0 disk.drbd1938.out0 disk.sda1.out0 disk.drbd1550.out0 disk.drbd1289.out0U2- _wandbruntime2297  _runtime2297G & lr-AdamW/pg10.0001327009882128411 & lr-AdamW/pg20.0001327009882128411 trainer/global_step5149 $ _timestamp1.785797584398425e+09 _runtime2297.221402021 _step408)W & lr-AdamW/pg10.0001327009882128411 & lr-AdamW/pg20.0001327009882128411 trainer/global_step5149 $ _timestamp1.785797584398425e+09 _runtime2297.221402021 _step408u_2- _wandbruntime2297  _runtime2297'O \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated88413372171264 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 4653711977984 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed34216027 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated14745577 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 trainer/global_step5149 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated90736643585700 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated34216029 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed88622475854848 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 4531198300672 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed88413355131904 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated88413372171264 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated92944657901568 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4531198300672 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed92944553432576 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 4531285730304 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed14745332 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated34216029 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed92944553432576 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated35023524 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 4520194420868 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed54380855 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4653842652160 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed88596882 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current247 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated93276343410688 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4520108617744 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed35023277 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed34216027 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 _runtime2297.222703703 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed86216432125472 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 % _timestamp1.7857975843994167e+09 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated92944657901568 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated54390511 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4531285730304 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0$FńO VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed88413355131904 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated88413372171264 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated92944657901568 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4531198300672 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed92944553432576 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 4531285730304 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed14745332 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated34216029 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed92944553432576 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated35023524 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 4520194420868 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed54380855 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4653842652160 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed88596882 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current247 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated93276343410688 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4520108617744 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed35023277 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed34216027 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 _runtime2297.222703703 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed86216432125472 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 % _timestamp1.7857975843994167e+09 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated92944657901568 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated54390511 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4531285730304 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated88413372171264 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 4653711977984 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed34216027 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated14745577 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 trainer/global_step5149 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated90736643585700 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated34216029 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed88622475854848 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 4531198300672 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121:12- _wandbruntime2297  _runtime2297P$&M REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated34222672 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated14748402 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 _step410 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed88614048 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated88639707875328 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated88623720 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed54391378 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed34222670 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed88639682971648 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 4521071894202 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _runtime2297.585689914 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed34222670 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4520986090966 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated92962704291328 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 4532077920256 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current236 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated92962704291328 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated54401048 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 # _timestamp1.78579758476235e+09 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 trainer/global_step5149 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 126472704 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed92962599815168 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed88614048 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated88430538934272 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated88623720 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed14748166 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 4532077920256 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed90754158388758 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed20281882 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 151376384 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed93294302158848 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated86233189337152 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed88430521894912 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4654745659904 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated35030286 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4532165357056 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed35030048 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated93294453535232 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 4654619187200 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121`9&M OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated35030286 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4532165357056 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed35030048 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated93294453535232 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 4654619187200 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated34222672 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated14748402 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 _step410 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed88614048 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated88639707875328 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated88623720 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed54391378 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed34222670 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed88639682971648 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 4521071894202 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _runtime2297.585689914 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed34222670 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4520986090966 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated92962704291328 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 4532077920256 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current236 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated92962704291328 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated54401048 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 # _timestamp1.78579758476235e+09 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 trainer/global_step5149 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 126472704 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed92962599815168 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed88614048 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated88430538934272 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated88623720 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed14748166 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 4532077920256 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed90754158388758 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed20281882 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 151376384 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed93294302158848 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated86233189337152 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed88430521894912 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4654745659904 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 j2- _wandbruntime2297  _runtime2297<2 2loss/SRqqt2/assignment_loss1.4479741429717377 trainer/global_step5149 % _timestamp1.7857975847640316e+09 %loss/total_loss19.82379012877952 /loss/FRt2/assignment_loss2.768974501777578 0loss/SRqqt1/assignment_loss1.17841553060632 1loss/FBt1/assignment_loss0.10243208303676732 .loss/FRt2/detection_loss2.566271475067845 0loss/SRqqt1/detection_loss2.044342656913949 /loss/FBt1/detection_loss1.0943885980683257 _step411 1loss/FBt2/assignment_loss0.22938116214105062 1loss/SRqqt2/detection_loss1.0396951039525113 epoch0 /loss/FBt2/detection_loss1.0046716758349075 0loss/FRt1/assignment_loss3.9727941992191167 /loss/FRt1/detection_loss2.3744489991894135 _runtime2297.586860851 0loss/SRqqt1/detection_loss2.044342656913949 /loss/FBt1/detection_loss1.0943885980683257 _step411 1loss/FBt2/assignment_loss0.22938116214105062 1loss/SRqqt2/detection_loss1.0396951039525113 epoch0 /loss/FBt2/detection_loss1.0046716758349075 0loss/FRt1/assignment_loss3.9727941992191167 /loss/FRt1/detection_loss2.3744489991894135 _runtime2297.586860851 2loss/SRqqt2/assignment_loss1.4479741429717377 trainer/global_step5149 % _timestamp1.7857975847640316e+09 %loss/total_loss19.82379012877952 /loss/FRt2/assignment_loss2.768974501777578 0loss/SRqqt1/assignment_loss1.17841553060632 1loss/FBt1/assignment_loss0.10243208303676732 .loss/FRt2/detection_loss2.566271475067845>MW(:  ݷU network.sent 4.708721e+06 disk.dm-0.in 2.47265625 disk.sda2.in0! proc.memory.rssMB 11260.40625$ disk./.usageGB578.3392639160156 disk.drbd1828.in0 disk.drbd1289.in0 disk.drbd2111.in0 disk.drbd1289.out0 disk.drbd1931.in0 disk.drbd1552.in0 disk.drbd1938.in0 disk.rbd0.in0 disk.dm-0.out94.79736328125 disk.sda2.out 62.87890625 disk.drbd1817.in0* proc.memory.availableMB38516.01953125 disk.drbd1226.in0 disk.rbd0.out0 disk.sda1.in0 network.recv327061 disk.drbd2041.out0 disk.drbd1673.in0 disk.drbd1828.out0 disk.drbd1226.out0 disk.drbd1584.out0 disk.drbd2118.out0 disk.rbd1.out 9.2314453125 disk.drbd1965.in0 proc.cpu.threads29 disk.drbd1673.out0 disk.sda1.out0 disk.drbd1965.out0* proc.memory.percent17.182016372680664 disk.drbd2111.out0 disk.drbd1938.out0 disk.drbd1550.in0$ memory_percent41.22921824455261 disk.drbd1510.out0 disk.drbd1356.in0 disk.drbd1356.out0 disk.rbd1.in0 cpu13.938496759435532 disk.drbd1552.out0 disk.drbd1510.in0 disk.drbd2078.in0 disk.drbd1584.in0 disk.sda.out 62.87890625) disk./.usagePercent32.34375921619473 disk.drbd1931.out0 disk.drbd2041.in0 disk.sda.in0# disk.nvme0n1.out94.79736328125 disk.drbd1817.out0 disk.drbd1550.out0 disk.drbd1285.in0 disk.drbd1285.out0 disk.drbd2078.out0 disk.drbd2118.in0 disk.nvme0n1.in 2.47265625Z(: ݷ՟V gpu.0.gpu35.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts129.307' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent43.102333333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors02- _wandbruntime2319  _runtime2319B6 & lr-AdamW/pg10.0001339895975371064 & lr-AdamW/pg20.0001339895975371064 trainer/global_step5199 % _timestamp1.7857976063705475e+09 _runtime2319.193530399 _step412 & lr-AdamW/pg20.0001339895975371064 trainer/global_step5199 % _timestamp1.7857976063705475e+09 _runtime2319.193530399 _step412 & lr-AdamW/pg10.0001339895975371064=F82- _wandbruntime2319  _runtime2319VwwO TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated34548179 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4699091051520 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated89271710321664 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed93846887718400 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed14889473 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 4698949891584 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed35364268 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed20474795 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed54908857 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated93846992187392 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated93846992187392 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed93846887718400 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 UGDeviceStatsMonitoeFFr.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed89271693282304 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 141159936 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4575281865728 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed34548177 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current259 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated89271710321664 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed94181781586432 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated91617540665208 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 _step413 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current257 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated34548179 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated54918513 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 $ _timestamp1.785797606371554e+09 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 trainer/global_step5199 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4563997081252 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated94181947650048 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed89457034 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated35364527 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated14889730 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed89482831694848 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 4575281865728 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 166063616 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed87053440741472 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4575194436096 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 _runtime2319.194838121 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated93846992187392 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated93846992187392 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed93846887718400 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed89271693282304 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 141159936 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4575281865728 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed34548177 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current259 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated89271710321664 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed94181781586432 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated91617540665208 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 _step413 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current257 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated34548179 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated54918513 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 $ _timestamp1.785797606371554e+09 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 trainer/global_step5199 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4563997081252 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated94181947650048 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed89457034 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated35364527 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated14889730 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed89482831694848 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 4575281865728 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 166063616 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed87053440741472 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4575194436096 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 _runtime2319.194838121 IDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated89288877084672 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current253 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated34554822 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed34554820 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed14892400 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 4564960357710 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated54929050 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated93865038577152 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated89483872 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed89474200 trainer/global_step5199 _step414 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed89474200 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 4576074055680 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed54919380 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated54929050 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated20478734 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4699984900608 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated35371387 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed89288860045312 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4576161492480 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed35371132 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated94200048615936 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed89500038811648 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 % _timestamp1.7857976067358336e+09 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed93864934100992 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated89483872 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed89288860045312 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 4576161492480 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current255 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated87070197953152 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated89483872 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed89474200 trainer/global_step5199 _step414 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed89474200 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 4576074055680 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed54919380 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated54929050 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated20478734 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4699984900608 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated35371387 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.curOCrent 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed89288860045312 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4576161492480 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed35371132 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated94200048615936 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed89500038811648 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 % _timestamp1.7857976067358336e+09 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed93864934100992 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated89483872 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed89288860045312 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 4576161492480 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current255 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated87070197953152 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated89288877084672 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current253 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated34554822 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed34554820 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed14892400 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 4564960357710 2- _wandbruntime2319  _runtime2319MOz 0loss/FRt1/assignment_loss4.0264348675464765 2loss/SRqqt2/assignment_loss1.7455952031199227 1loss/FBt1/assignment_loss0.07667787170701104 /loss/FBt1/detection_loss1.8472791773447852 /loss/FBt2/detection_loss0.8933098614215851 &loss/total_loss21.678912148059204 _runtime2319.560287711 _step415 0loss/FRt2/assignment_loss3.1716125030671396 /loss/FRt2/detection_loss2.4653610263860997 epoch0 trainer/global_step5199 % _timestamp1.7857976067374656e+09 0loss/FBt2/assignment_loss0.4932319224390085 1loss/SRqqt1/assignment_loss1.474822436273098 /loss/FRt1/detection_loss2.2450983144085983 1loss/SRqqt1/detection_loss2.2402188907416423 1loss/SRqqt2/detection_loss0.99927007360383870  1loss/SRqqt1/detection_loss2.2402188907416423 1loss/SRqqt2/detection_loss0.9992700736038387 0loss/FRt1/assignment_loss4.0264348675464765 2loss/SRqqt2/assignment_loss1.7455952031199227 1loss/FBt1/assignment_loss0.07667787170701104 /loss/FBt1/detection_loss1.8472791773447852 /loss/FBt2/detection_loss0.8933098614215851 &loss/total_loss21.678912148059204 _runtime2319.560287711 _step415 0loss/FRt2/assignment_loss3.1716125030671396 /loss/FRt2/detection_loss2.4653610263860997 epoch0 trainer/global_step5199 % _timestamp1.7857976067374656e+09 0loss/FBt2/assignment_loss0.4932319224390085 1loss/SRqqt1/assignment_loss1.474822436273098 /loss/FRt1/detection_loss2.2450983144085983yݶ(: ·U gpu.0.gpu30.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts111.005' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent37.001666666666665 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0}ĥP(:  U disk.drbd1965.in0 disk.drbd1550.in0 disk.sda.in0 disk.drbd1510.in0 disk.drbd2111.out0 disk.drbd1828.out0# disk.nvme0n1.out97.12548828125 disk.drbd1356.in0 network.sent 4.759214e+06 disk.drbd1938.out0 disk.sda1.in0 disk.dm-0.in 2.47265625 disk.drbd1584.out0% memory_percent41.230082511901855 disk.drbd1510.out0 disk.rbd0.in0 disk.nvme0n1.in 2.47265625 disk.drbd2111.in0* proc.memory.percent17.182016372680664 disk.drbd1817.in0 network.recv330062 disk.sda.out 63.234375) disk./.usagePercent32.34374748261348 disk.drbd2078.in0( proc.memory.availableMB 38515.453125 disk.drbd1285.out0 disk.drbd1226.in0 disk.drbd2118.in0 disk.drbd1356.out0 disk.sda2.out 63.234375 proc.cpu.threads29 disk.drbd1552.in0 disk.drbd1289.in0 disk.drbd1226.out0 disk.rbd1.out 9.263671875 disk.drbd1673.out0 disk.drbd1931.in0 disk.drbd1552.out0 disk.drbd1550.out0! proc.memory.rssMB 11260.40625 disk.drbd1965.out0 disk.drbd1673.in0# disk./.usageGB578.339054107666 disk.rbd0.out0 disk.drbd1817.out0 disk.rbd1.in0 cpu13.936005956641141 disk.drbd1285.in0 disk.dm-0.out97.12548828125 disk.sda2.in0 disk.drbd1931.out0 disk.drbd2041.out0 disk.drbd1289.out0 disk.drbd1938.in0 disk.sda1.out0 disk.drbd2118.out0 disk.drbd2041.in0 disk.drbd1828.in0 disk.drbd2078.out0 disk.drbd1584.in0L"(: U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts132.175' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent44.05833333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0#zsT(:  U disk.rbd1.in0 disk.nvme0n1.in 2.47265625 disk.drbd2111.in0 disk.drbd2118.in0! proc.memory.rssMB 11260.40625 disk.drbd1965.out0 disk.dm-0.in 2.47265625 disk.drbd1931.in0 disk.drbd2078.in0 disk.drbd1817.out0 disk.drbd1965.in0 disk.drbd1552.out0 disk.rbd0.in0 disk.drbd2118.out0 disk.sda2.out 63.7265625* proc.memory.percent17.182016372680664 disk.drbd1356.in0$ memory_percent41.23132824897766 disk.drbd1550.out0 disk.dm-0.out97.39404296875 disk.drbd1226.in0# disk.nvme0n1.out97.39404296875 cpu13.933653244956346$ disk./.usageGB578.3386192321777 disk.drbd1673.out0 network.sent 4.759364e+06 disk.drbd1289.in0 disk.drbd1938.in0 disk.drbd1938.out0 disk.drbd1931.out0 disk.drbd1828.in0 disk.drbd1584.out0 disk.drbd2111.out0 disk.sda1.out0 disk.drbd1285.out0 disk.drbd2078.out0* proc.memory.availableMB38514.63671875 disk.drbd1673.in0 disk.drbd1552.in0 proc.cpu.threads29 disk.drbd1285.in0 disk.drbd1226.out0 network.recv330212 disk.drbd2041.out0 disk.sda.in0 disk.sda1.in0 disk.drbd1356.out0 disk.rbd1.out 9.357421875 disk.drbd1289.out0 disk.rbd0.out0 disk.drbd1510.out0 disk.drbd1584.in0 disk.sda2.in0 disk.drbd1817.in0 disk.sda.out 63.7265625 disk.drbd2041.in0 disk.drbd1828.out0 disk.drbd1550.in0 disk.drbd1510.in0) disk./.usagePercent32.34372316209962<2- _wandbruntime2341  _runtime2341FJ trainer/global_step5249 % _timestamp1.7857976282539766e+09 _runtime2341.076956263 _step416 ' lr-AdamW/pg10.00013527820686137172 ' lr-AdamW/pg20.000135278206861371728a trainer/global_step5249 % _timestamp1.7857976282539766e+09 _runtime2341.076956263 _step416 ' lr-AdamW/pg10.00013527820686137172 ' lr-AdamW/pg20.000135278206861371722- _wandbruntime2341  _runtime2341oF'O H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed90130031432704 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed90343187534848 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated34880329 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4619278001152 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated55446515 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed34880327 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated87890466396832 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed92498334902232 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed90317186 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated55446515 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 trainer/global_step5249 % _timestamp1.7857976282549348e+09 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 4619278001152 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4619190571520 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated90326844 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 _step417 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated94749326473216 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated90343212438528 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed94749222004224 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed55436859 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated20671647 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4744262196224 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated90130048472064 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 4744135716352 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed94749222004224 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed35705160 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed20671645 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed90130031432704 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 4619190571520 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887R=X'O E8DeviceStatsMonitor.on_train_batch_start/active.all.freed90317186 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated55446515 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 trainer/global_step5249 % _timestamp1.7857976282549348e+09 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 4619278001152 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4619190571520 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated90326844 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 _step417 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated94749326473216 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated90343212438528 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed94749222004224 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed55436859 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated20671647 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4744262196224 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated90130048472064 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 4744135716352 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed94749222004224 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed35705160 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed20671645 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed90130031432704 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 4619190571520 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed90130031432704 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed90343187534848 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated34880329 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4619278001152 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated55446515 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed34880327 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated87890466396832 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed9249833490223242- _wandbruntime2341  _runtime2341M VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated94767372862976 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed94767268386816 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed90334352 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated92516055390370 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed925159525477 8#74 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 4608848821218 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated90344024 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 4745033930752 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated90147215235072 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated15036547 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated90360419555328 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed34886970 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4620157627904 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated34886972 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current247 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated87907206569152 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed87907189529792 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated35712131 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 _runtime2341.455716391 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed90147198195712 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated34886972 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated55457052 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4745170889216 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated94767372862976 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated55457052 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed90334352 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed35711884 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed20675582 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed90360394651648 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 _step418 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 4620157627904 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed90147198195712 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 4620070191104 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed34886970 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed15036302 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed55447382 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current245 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 4620070191104 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated95105590444544 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 % _timestamp1.7857976286323087e+09 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4608763017982 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 trainer/global_step5249 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated90344024 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed55447382 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed95105428582400X%&M =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated90344024 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed55447382 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed95105428582400 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated94767372862976 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed94767268386816 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed90334352 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated92516055390370 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed92515952547774 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 4608848821218 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated90344024 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 4745033930752 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated90147215235072 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated15036547 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated90360419555328 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed34886970 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4620157627904 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated34886972 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current247 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated87907206569152 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed87907189529792 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated35712131 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 _runtime2341.455716391 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed90147198195712 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated34886972 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated55457052 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4745170889216 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated94767372862976 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated55457052 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed90334352 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed35711884 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed20675582 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed90360394651648 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 _step418 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 4620157627904 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed90147198195712 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 4620070191104 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed34886970 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed15036302 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed55447382 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current245 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 4620070191104 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated95105590444544 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 % _timestamp1.7857976286323087e+09 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4608763017982 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 trainer/global_step5249Vx2- _wandbruntime2341  _runtime2341  1loss/FBt2/assignment_loss0.20269610353661846 /loss/SRqqt1/detection_loss2.03484749342456 epoch0 trainer/global_step5249 _step419 /loss/FRt1/assignment_loss3.757002568244934 0loss/FRt2/assignment_loss2.8135322641867857 1loss/SRqqt1/assignment_loss1.430119260242491 2loss/SRqqt2/assignment_loss1.6337058603922952 .loss/FRt2/detection_loss2.571768454920787 1loss/SRqqt2/detection_loss0.9622433529320088 /loss/FBt1/detection_loss1.5632524647980146 % _timestamp1.7857976286335523e+09 .loss/FRt1/detection_loss2.732499356830821 .loss/FBt2/detection_loss4.669666975736618 &loss/total_loss24.409959187871834 1loss/FBt1/assignment_loss0.03862503262589995 _runtime2341.456467266&\ 1loss/FBt2/assignment_loss0.20269610353661846 /loss/SRqqt1/detection_loss2.03484749342456 epoch0 trainer/global_step5249 _step419 /loss/FRt1/assignment_loss3.757002568244934 0loss/FRt2/assignment_loss2.8135322641867857 1loss/SRqqt1/assignment_loss1.430119260242491 2loss/SRqqt2/assignment_loss1.6337058603922952 .loss/FRt2/detection_loss2.571768454920787 1loss/SRqqt2/detection_loss0.9622433529320088 /loss/FBt1/detection_loss1.5632524647980146 % _timestamp1.7857976286335523e+09 .loss/FRt1/detection_loss2.732499356830821 .loss/FBt2/detection_loss4.669666975736618 &loss/total_loss24.409959187871834 1loss/FBt1/assignment_loss0.03862503262589995 _runtime2341.456467266 .<(: ęU gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp43.0 gpu.0.powerWatts128.301' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent42.766999999999996 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0T(:  U disk.drbd1817.in0 disk.drbd1938.out0$ disk./.usageGB578.3404846191406 disk.drbd1552.in0! proc.memory.rssMB 11260.40625 disk.drbd1510.out0 disk.drbd2118.out0 disk.nvme0n1.in 2.47265625 disk.drbd1673.in0 disk.drbd1584.out0 disk.drbd1552.out0 disk.drbd1828.in0 proc.cpu.threads29 disk.drbd2111.in0 disk.drbd2041.out0 disk.drbd1550.in0 disk.sda.out 64.16015625 disk.drbd2078.in0# disk.nvme0n1.out98.10791015625 disk.sda2.out 64.16015625 disk.drbd1226.in0 disk.drbd1356.in0 disk.rbd1.in0 disk.drbd1965.in0 disk.rbd0.in0$ memory_percent41.23063087463379* proc.memory.percent17.182016372680664 cpu13.931431904663699 disk.rbd1.out 9.419921875 disk.rbd0.out0' proc.memory.availableMB 38515.09375* disk./.usagePercent32.343827484303795 disk.sda1.out0 disk.drbd1285.in0 disk.drbd1931.out0 disk.drbd1289.out0 disk.drbd1931.in0 network.recv334649 disk.dm-0.in 2.47265625 disk.drbd1673.out0 disk.sda.in0 disk.drbd1285.out0 disk.drbd1356.out0 disk.drbd2111.out0 disk.drbd1938.in0 disk.drbd1510.in0 disk.drbd1828.out0 disk.drbd1550.out0 disk.drbd1289.in0 network.sent 4.849372e+06 disk.drbd2078.out0 disk.drbd1584.in0 disk.drbd1965.out0 disk.drbd1226.out0 disk.drbd2118.in0 disk.drbd1817.out0 disk.drbd2041.in0 disk.dm-0.out98.10791015625 disk.sda2.in0 disk.sda1.in02- _wandbruntime2363  _runtime23639g$ % lr-AdamW/pg10.000136566816185637 % lr-AdamW/pg20.000136566816185637 trainer/global_step5299 % _timestamp1.7857976501887116e+09 _runtime2363.011677838 _step420<. _runtime2363.011677838 _step420 % lr-AdamW/pg10.000136566816185637 % lr-AdamW/pg20.000136566816185637 trainer/global_step5299 % _timestamp1.7857976501887116e+09p2- _wandbruntime2363  _runtime2363$$O =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated95993044282880 % _timestamp1.7857976501896307e+09 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 4663186706944 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed35212477 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4663274136576 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed90988369583104 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated55974517 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current231 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed95992897093632 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed90988369583104 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed20868495 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated15176406 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 4651859811392 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current229 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated95651660759040 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated35212479 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed95651556290048 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed55964861 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed35212477 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed93379231981740 trainer/global_step5299 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated95651660759040 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated35212479 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4789476004352 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4651774008268 _step421 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated91186996 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0F'O RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated95651660759040 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated35212479 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed95651556290048 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed55964861 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed35212477 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed93379231981740 trainer/global_step5299 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated95651660759040 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated35212479 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4789476004352 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4651774008268 _step421 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated91186996 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated95993044282880 % _timestamp1.7857976501896307e+09 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 4663186706944 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed35212477 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4663274136576 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed90988369583104 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated55974517 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current231 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed95992897093632 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed90988369583104 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed20868495 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated15176406 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 4651859811392 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current229 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130=2- _wandbruntime2363  _runtime2363*&M ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4664153763328 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current250 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed91005536346112 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 % _timestamp1.7857976505590348e+09 _step422 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed96011007913472 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated96011165581312 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4790390185984 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed93396849627282 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 4664066326528 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated91005553385472 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated20872434 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed88744198145792 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated91204176 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated55985054 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed36051410 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 4664153763328 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated15179228 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed15178978 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated36051662 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current252 _runtime2363.382370934 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated88744215185152 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed20872432 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed91194504 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed91220750491648 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated91204176 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed55975384 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated95669707148800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated35219122 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed95669602672640 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed35219120 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 4652737284726 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated35219122 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated91005553385472 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated95669707148800 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4652651481490 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated55985054 c&M VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed91005536346112 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 % _timestamp1.7857976505590348e+09 _step422 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed96011007913472 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated96011165581312 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4790390185984 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed93396849627282 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 4664066326528 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated91005553385472 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated20872434 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed88744198145792 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated91204176 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated55985054 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed36051410 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 4664153763328 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated15179228 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed15178978 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated36051662 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current252 _runtime2363.382370934 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated88744215185152 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed20872432 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed91194504 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed91220750491648 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated91204176 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed55975384 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated95669707148800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated35219122 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed95669602672640 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed35219120 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 4652737284726 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated35219122 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated91005553385472 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated95669707148800 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4652651481490 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated55985054 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4664153763328 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current250 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160V2- _wandbruntime2363  _runtime2363eQT 2loss/SRqqt2/assignment_loss1.5416616106575185 .loss/FRt2/detection_loss2.345771093970873 1loss/SRqqt1/detection_loss2.4296358942308207 _step423 1loss/FBt2/assignment_loss0.37629457710924424 /loss/FRt1/detection_loss2.5157190596169614 1loss/SRqqt2/detection_loss1.0525968808532609 .loss/FBt2/detection_loss38.57762471834818 %loss/total_loss58.66586332061502 % _timestamp1.7857976505609696e+09 _runtime2363.383798894 0loss/FRt1/assignment_loss3.9149124051662203 0loss/FRt2/assignment_loss3.1826947482538896 1loss/FBt1/assignment_loss0.09072950287528794 .loss/FBt1/detection_loss1.334099801097267 trainer/global_step5299 epoch0 2loss/SRqqt1/assignment_loss1.3041230284354903E=  2loss/SRqqt1/assignment_loss1.3041230284354903 2loss/SRqqt2/assignment_loss1.5416616106575185 .loss/FRt2/detection_loss2.345771093970873 1loss/SRqqt1/detection_loss2.4296358942308207 _step423 1loss/FBt2/assignment_loss0.37629457710924424 /loss/FRt1/detection_loss2.5157190596169614 1loss/SRqqt2/detection_loss1.0525968808532609 .loss/FBt2/detection_loss38.57762471834818 %loss/total_loss58.66586332061502 % _timestamp1.7857976505609696e+09 _runtime2363.383798894 0loss/FRt1/assignment_loss3.9149124051662203 0loss/FRt2/assignment_loss3.1826947482538896 1loss/FBt1/assignment_loss0.09072950287528794 .loss/FBt1/detection_loss1.334099801097267 trainer/global_step5299 epoch0 (: U gpu.0.gpu6.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp43.0 gpu.0.powerWatts111.758' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercenS^t37.25266666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0%R(:  U) disk./.usagePercent32.34383068437141 disk.drbd1226.out0 disk.drbd2118.in0 disk.drbd1931.in0 disk.drbd1289.out0 disk.sda.in0* proc.memory.availableMB38514.56640625 disk.drbd1285.in0 disk.sda2.out 64.4140625 disk.sda.out 64.4140625 disk.dm-0.out 98.3486328125 disk.drbd2041.in0 disk.drbd2041.out0" disk.nvme0n1.out 98.3486328125 disk.drbd1226.in0 disk.drbd2111.in0 disk.drbd1828.out0 disk.drbd1285.out0 disk.drbd1356.out0$ disk./.usageGB578.3405418395996 disk.drbd1938.out0 disk.drbd2118.out0 disk.rbd1.in0 disk.drbd1510.in0 disk.dm-0.in 2.47265625 disk.drbd1931.out0! proc.memory.rssMB 11260.40625 proc.cpu.threads29 disk.drbd1289.in0 disk.drbd1510.out0 disk.drbd1550.in0 network.sent 4.850477e+06 disk.drbd1938.in0 disk.drbd2078.in0 disk.drbd2111.out0 disk.drbd1817.in0 disk.rbd0.out0 disk.drbd1356.in0 disk.drbd1817.out0 disk.rbd1.out 9.419921875 disk.drbd1673.in0 disk.rbd0.in0 disk.drbd1584.in0* proc.memory.percent17.182016372680664 cpu13.929034264305097 disk.sda1.out0 disk.drbd2078.out0 disk.drbd1584.out0 disk.nvme0n1.in 2.47265625 disk.drbd1965.in0 disk.drbd1828.in0 disk.sda1.in0 disk.sda2.in0 disk.drbd1965.out0 disk.drbd1673.out0 disk.drbd1552.out0 disk.drbd1550.out0$ memory_percent41.23143553733826 disk.drbd1552.in0 network.recv335506~(: ۥU gpu.0.gpu0.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp43.0 gpu.0.powerWatts129.908' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent43.30266666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0ޅvT(:  V disk.dm-0.out99.03466796875 disk.drbd1817.in0 disk.drbd1552.out0 disk.rbd0.in0! proc.memory.rssMB 11260.40625 disk.drbd1828.out0 disk.drbd1289.out0 disk.drbd2111.out0 disk.nvme0n1.in 2.47265625 disk.sda2.in0 disk.drbd1938.in0 cpu13.92675691780717$ disk./.usageGB578.3405609130859 disk.rbd1.out 9.5380859375$ memory_percent41.23184680938721 disk.drbd1965.in0 disk.drbd1226.out0 disk.sda2.out 64.83984375 disk.drbd2111.in0 disk.sda.in0 disk.sda.out 64.83984375 network.recv335614 disk.drbd1584.in0# disk.nvme0n1.out99.03466796875 disk.drbd1289.in0 disk.drbd1510.out0 disk.dm-0.in 2.47265625 disk.drbd1550.out0 disk.sda1.out0 disk.drbd1226.in0 disk.drbd2078.out0 disk.drbd2041.in0 disk.drbd1510.in0 disk.drbd2078.in0 disk.drbd1584.out0 disk.drbd2118.in0 disk.drbd1965.out0 disk.drbd1931.in0 disk.drbd1673.out0 disk.drbd1931.out0 proc.cpu.threads29 disk.drbd1285.in0 disk.drbd1285.out0 disk.drbd2118.out0 disk.rbd1.in0 disk.drbd2041.out0 disk.drbd1828.in0 disk.drbd1550.in0 disk.drbd1938.out0 disk.sda1.in0* proc.memory.percent17.182016372680664 disk.drbd1817.out0 network.sent 4.850585e+06) disk./.usagePercent32.34383175106061 disk.drbd1356.in0 disk.drbd1356.out0 disk.drbd1673.in0 disk.rbd0.out0( proc.memory.availableMB 38514.296875 disk.drbd1552.in002- _wandbruntime2385  _runtime2385 & lr-AdamW/pg20.0001378554255099023 trainer/global_step5349 $ _timestamp1.785797672284322e+09 _runtime2385.107318183 _step424 & lr-AdamW/pg10.0001378554255099023q & lr-AdamW/pg10.0001378554255099023 & lr-AdamW/pg20.0001378554255099023 trainer/global_step5349 $ _timestamp1.785797672284322e+09 _runtime2385.107318183 _step424E9a2- _wandbruntime2385  _runtime2385!E'O N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 _step425 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current272 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed21065345 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed92063899214848 trainer/global_step5349 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated96898588915200 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 % _timestamp1.7857976722852798e+09 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated96553995044864 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current270 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 4707270272000 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed96553890575872 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated96553995044864 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed36384761 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated21065347 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed96898427045888 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed35544627 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4834664796672 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated36385033 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated56502519 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated91846724772864 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated15319686 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated94260231903732 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed89564466589472 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed92037490 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed35544627 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed96553890575872 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated56502519 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed91846707733504 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4707182842368 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed91846707733504 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed56492863 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121%'O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 4834527831040 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed15319416 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed94260129061248 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed56492863 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated92047148 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated92063924118528 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4695662471776 _runtime2385.111664132 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 4707182842368 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 _step425 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current272 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed21065345 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed92063899214848 trainer/global_step5349 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated96898588915200 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 % _timestamp1.7857976722852798e+09 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated96553995044864 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current270 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 4707270272000 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed96553890575872 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated96553995044864 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed36384761 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated21065347 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed96898427045888 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed35544627 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4834664796672 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated36385033 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated56502519 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated91846724772864 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated15319686 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated94260231903732 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed89564466589472 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed92037490 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed35544627 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed96553890575872 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated56502519 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed91846707733504 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4707182842368 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed91846707733504 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed56492863 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 I6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated94277849549386 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 _step426 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 4696625748234 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed92054656 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated56513056 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed56503386 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current255 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed92054656 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated35551272 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated21069284 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated96572041434624 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 4708149898752 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated15322571 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated89581223801152 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4708149898752 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed56503386 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated35551272 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4696539944998 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed21069282 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current253 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 163959296 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated92064328 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed36391600 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated56513056 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 4708062461952 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4835568852992 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated94277849549386 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 _step426 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 4696625748234 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed92054656 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated56513056 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed56503386 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current255 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed92054656 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated35551272 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated21069284 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated96572041434624 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 4708149898752 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated15322571 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated89581223801152 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4708149898752 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed56503386 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated35551272 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4696539944998 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed21069282 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current253 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 163959296 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated92064328 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed36391600 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated56513056 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 4708062461952 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4835568852992 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed57020865 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4739550935284 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated21262197 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed97804018555904 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed35876777 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed36725939 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current232 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed35876777 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed97456224861696 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated57030521 trainer/global_step5399 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4751266407424 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated92907300 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current232 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed35876777 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed97456224861696 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated57030521 trainer/global_step5399 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4751266407424 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated92907300 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed57020865 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4739550935284 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated21262197 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed97804018555904 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed35876777 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed36725939 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _step430 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed57031388 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed92722212646912 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated97474375720448 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed97474271244288 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 4880676280832 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated95158746628894 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 4752146034176 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 4752058597376 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed92722212646912 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated90418232417152 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 4740514211742 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 4752058597376 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated92924480 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated15466899 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated97822300314624 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated57041058 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated21266134 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 _runtime2406.907589326 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed97822138452480 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed36732771 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4740428408506 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 % _timestamp1.7857976940842516e+09 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed92914808 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed57031388 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current262 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated92722229686272 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current260 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed92914808 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed15466639 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated92722229686272 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated35883422 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated36733033 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed92941462171648 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated57041058 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed35883420 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated92924480 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated15466899 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated97822300314624 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated57041058 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated21266134 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 _runtime2406.907589326 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed97822138452480 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed36732771 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4740428408506 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 % _timestamp1.7857976940842516e+09 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed92914808 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed57031388 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current262 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated92722229686272 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current260 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed92914808 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed15466639 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated92722229686272 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated35883422 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated36733033 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed92941462171648 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated57041058 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed35883420 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _step430Z2- _wandbruntime2406  _runtime2406R%O 1loss/FBt1/assignment_loss0.06933683883883572 1loss/FBt2/assignment_loss0.40939735857694937 0loss/SRqqt1/detection_loss2.282741236741896 0loss/SRqqt2/detection_loss1.050442360113672 _runtime2406.908670931 .loss/FRt1/assignment_loss3.98923116571763 0loss/FRt2/assignment_loss2.6038325754078953 2loss/SRqqt1/assignment_loss1.4290354958286993 .loss/FRt1/detection_loss2.344279468560419 .loss/FRt2/detection_loss2.504937469959259 /loss/FBt2/detection_loss1.6826767813075671 &loss/total_loss20.980111000901136 epoch0 2loss/SRqqt2/assignment_loss1.6054836789423481 /loss/FBt1/detection_loss1.0087165709059642 _step431 trainer/global_step5399 % _timestamp1.7857976940858495e+09ԣJX .loss/FRt1/assignment_loss3.98923116571763 0loss/FRt2/assignment_loss2.6038325754078953 2loss/SRqqt1/assignment_loss1.4290354958286993 .loss/FRt1/detection_loss2.344279468560419 .loss/FRt2/detection_loss2.504937469959259 /loss/FBt2/detection_loss1.6826767813075671 &loss/total_loss20.980111000901136 epoch0 2loss/SRqqt2/assignment_loss1.6054836789423481 /loss/FBt1/detection_loss1.0087165709059642 _step431 trainer/global_step5399 % _timestamp1.7857976940858495e+09 1loss/FBt1/assignment_loss0.06933683883883572 1loss/FBt2/assignment_loss0.40939735857694937 0loss/SRqqt1/detection_loss2.282741236741896 0loss/SRqqt2/detection_loss1.050442360113672 _runtime2406.908670931j:(: ƸU gpu.0.gpu30.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts100.547' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent33.51566666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0qY(:  ƸU disk.drbd2118.in0 disk.drbd1673.in0% memory_percent41.231220960617065 disk.drbd1510.out0 disk.drbd1552.out0 disk.drbd1828.out0 disk.drbd1550.in0 disk.sda.in0 disk.dm-0.in 2.47265625 disk.drbd1226.out0 disk.drbd2118.out0 disk.rbd1.in0 disk.drbd1965.in0 disk.rbd0.out0 disk.drbd1817.out0 disk.sda1.out0 disk.drbd1285.out0* proc.memory.availableMB38514.70703125 cpu13.92316560411184 disk.drbd1226.in0 disk.drbd2078.out0$ disk.nvme0n1.out100.03857421875 disk.sda2.out 65.60546875 disk.drbd1817.in0 disk.drbd1931.in0 disk.drbd1552.in0 disk.drbd1584.out0 disk.sda2.in0 disk.drbd2111.in0 disk.drbd1965.out0 disk.drbd2041.out0 disk.drbd1550.out0! disk.dm-0.out100.03857421875 disk.drbd1356.in0 disk.rbd1.out 9.7802734375 disk.drbd1289.in0 disk.rbd0.in0 disk.nvme0n1.in 2.47265625 disk.drbd1931.out0 disk.drbd1938.out0 disk.sda.out 65.60546875* proc.memory.percent17.182016372680664 network.sent 4.966253e+06 disk.drbd1584.in0 disk.sda1.in0 disk.drbd1673.out0 disk.drbd1289.out0! proc.memory.rssMB 11260.40625) disk./.usagePercent32.34383132438493 disk.drbd1356.out0 proc.cpu.threads29 disk.drbd2041.in0 disk.drbd1938.in0 network.recv340656$ disk./.usageGB578.3405532836914 disk.drbd2111.out0 disk.drbd1828.in0 disk.drbd1285.in0 disk.drbd1510.in0 disk.drbd2078.in0ګb2- _wandbruntime2428  _runtime2428 _runtime2428.737945491 _step432 & lr-AdamW/pg10.0001404326441584329 & lr-AdamW/pg20.0001404326441584329 trainer/global_step5449 % _timestamp1.7857977159149876e+09S< & lr-AdamW/pg10.0001404326441584329 & lr-AdamW/pg20.0001404326441584329 trainer/global_step5449 % _timestamp1.7857977159149876e+09 _runtime2428.737945491 _step432F2q2- _wandbruntime2428  _runtime2428 XO ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated98709753735680 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4783439398792 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated98358663616512 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed98358559147520 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed21459045 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current245 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 4924983068672 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated21459047 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed98358559147520 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4795262542848 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated93767452 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed91238483821472 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated36208929 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed96021923220264 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated98358663616512 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4795175113216 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated93563401073664 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current243 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 4783525201916 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated93563401073664 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated15608349 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated91238500860832 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 E![e?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed98709593963520 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 4795175113216 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4925117937152 trainer/global_step5449 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed57548867 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated96022026062748 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 $ _timestamp1.785797715915852e+09 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 4795262542848 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated93784635798528 _step433 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed93784610894848 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated36208929 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated57558523 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 _runtime2428.742672091 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated57558523 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484u-'O GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed36208927 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 _runtime2428.742672091 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated57558523 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated98709753735680 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4783439398792 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated98358663616512 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed98358559147520 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed21459045 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current245 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 4924983068672 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated21459047 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed98358559147520 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4795262542848 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated93767452 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed91238483821472 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated36208929 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed96021923220264 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated98358663616512 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4795175113216 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated93563401073664 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current243 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 4783525201916 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated93563401073664 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated15608349 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated91238500860832 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed98709593963520 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 4795175113216 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4925117937152 trainer/global_step5449 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed57548867 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated96022026062748 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 $ _timestamp1.785797715915852e+09 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 4795262542848 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated93784635798528 _step433 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed93784610894848 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated36208929 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated57558523 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated93784632 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 % _timestamp1.7857977162806084e+09 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated93580567836672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 trainer/global_step5449 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed93580550797312 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed98727709405696 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4784316872014 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _step434 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed37073796 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated98376710006272 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed21462982 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated36215572 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated21462984 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated57569060 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated98727860782080 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 _runtime2429.103937883 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 4925891394048 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated91255241033152 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated98376710006272 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 4796054732800 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 151376384 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed93801818011648 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated96039643708402 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed36215570 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4796142169600 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 4784402675250 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated93784632 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated93801842915328 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed98376605530112 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360mZgM G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated93784632 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated93801842915328 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed98376605530112 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed15610814 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed93774960 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated36215572 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed93774960 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated93784632 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 % _timestamp1.7857977162806084e+09 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated93580567836672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 trainer/global_step5449 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed93580550797312 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed98727709405696 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4784316872014 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _step434 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed37073796 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated98376710006272 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed21462982 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated36215572 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated21462984 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated57569060 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated94  8727860782080 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 _runtime2429.103937883 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 4925891394048 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated91255241033152 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated98376710006272 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 4796054732800 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 151376384 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed93801818011648 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated96039643708402 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed36215570 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4796142169600 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 478440267525002- _wandbruntime2429  _runtime2429|8/ 1loss/FBt1/assignment_loss0.11996254462083071 /loss/FRt1/detection_loss2.2358968831783486 .loss/FRt2/detection_loss2.626040197081036 0loss/SRqqt2/detection_loss1.064379693498569 /loss/FBt2/detection_loss0.4071179618300797 %loss/total_loss19.90295768191026 _step435 /loss/FRt1/assignment_loss4.104665869523671 trainer/global_step5449 _runtime2429.105077355 1loss/SRqqt1/detection_loss1.8590616278131218 epoch0 $ _timestamp1.785797716282249e+09 .loss/FRt2/assignment_loss2.75278631956489 2loss/SRqqt1/assignment_loss1.1519559122183745 1loss/SRqqt2/assignment_loss1.649506380047117 0loss/FBt2/assignment_loss0.5161023794540337 /loss/FBt1/detection_loss1.4154819130801894]KHg _step435 /loss/FRt1/assignment_loss4.104665869523671 trainer/global_step5449 _runtime2429.105077355 1loss/SRqqt1/detection_loss1.8590616278131218 epoch0 $ _timestamp1.785797716282249e+09 .loss/FRt2/assignment_loss2.75278631956489 2loss/SRqqt1/assignment_loss1.1519559122183745 1loss/SRqqt2/assignment_loss1.649506380047117 0loss/FBt2/assignment_loss0.5161023794540337 /loss/FBt1/detection_loss1.4154819130801894 1loss/FBt1/assignment_loss0.11996254462083071 /loss/FRt1/detection_loss2.2358968831783486 .loss/FRt2/detection_loss2.626040197081036 0loss/SRqqt2/detection_loss1.064379693498569 /loss/FBt2/detection_loss0.4071179618300797 %loss/total_loss19.90295768191026pl)(: ո튂U gpu.0.gpu30.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp43.0 gpu.0.powerWatts129.658' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent43.219333333333324 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0Q(:  ոU* proc.memory.percent17.182016372680664! disk.dm-0.out100.45263671875 disk.rbd0.out0 disk.sda.in0 disk.sda2.in0! proc.memory.rssMB 11260.40625 proc.cpu.threads29$ disk.nvme0n1.out100.45263671875 disk.drbd2041.out0 network.recv341687 disk.sda.out66.1875 disk.drbd2111.out0 disk.drbd1828.in0 disk.drbd1817.in0 disk.drbd1673.out0 cpu13.920827128639669 disk.drbd1828.out0 disk.drbd1510.out0 disk.drbd2078.out0 disk.drbd2118.in0 disk.drbd1931.in0 disk.drbd1938.out0 disk.drbd1584.in0 disk.drbd1584.out0 disk.drbd2118.out0 disk.drbd1673.in0 disk.drbd1931.out0$ disk./.usageGB578.3406143188477 disk.drbd1938.in0 disk.nvme0n1.in 2.47265625 network.sent 4.967266e+06 disk.drbd1550.in0 disk.drbd1285.in0 disk.drbd1226.in0) disk./.usagePercent32.34383473779038 disk.drbd1285.out0 disk.drbd1226.out0 disk.rbd1.in0 disk.drbd1289.in0 disk.drbd1817.out0 disk.drbd1965.in0* proc.memory.availableMB38514.41015625 disk.drbd1552.in0 disk.drbd1550.out0 disk.drbd1289.out0 disk.drbd1510.in0 disk.drbd2041.in0 disk.rbd0.in0 disk.sda1.out0 disk.drbd2078.in0 disk.rbd1.out 9.7802734375$ memory_percent41.23167395591736 disk.drbd1552.out0 disk.drbd1356.in0 disk.sda2.out66.1875 disk.drbd2111.in0 disk.drbd1965.out0 disk.sda1.in0 disk.dm-0.in 2.47265625 disk.drbd1356.out0z(: ԮU gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts128.282' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent42.760666666666665 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0{2P(:  U disk.drbd1289.in0 disk.drbd1584.out0 disk.nvme0n1.in 2.47265625# disk.nvme0n1.out100.7841796875 disk.drbd1931.in0 disk.drbd2041.out0 disk.sda.out 66.5078125* proc.memory.percent17.182016372680664 disk.sda2.out 66.5078125 disk.rbd1.in0 disk.drbd1931.out0 disk.drbd1828.in0 disk.drbd2118.out0 disk.drbd1356.out0* disk./.usagePercent32.343864178412424 disk.drbd1673.out0 disk.drbd1552.in0 disk.drbd1289.out0 disk.sda1.out0 disk.drbd2111.out0 disk.drbd1817.in0 disk.drbd1817.out0 disk.drbd1673.in0 disk.drbd1550.in0 disk.sda.in0 proc.cpu.threads29 disk.drbd1510.in0 disk.dm-0.in 2.47265625 disk.rbd1.out9.8125* proc.memory.availableMB38514.65234375 disk.rbd0.in0 disk.drbd1828.out0 disk.drbd2078.in0 disk.drbd1584.in0 cpu13.917811225190928 disk.drbd1938.in0 disk.dm-0.out100.7841796875 disk.drbd1226.out0 disk.sda2.in0 disk.drbd2111.in0 disk.drbd1965.in0 disk.rbd0.out0 disk.drbd1285.in0 disk.drbd1510.out0 disk.drbd1226.in0$ disk./.usageGB578.3411407470703 disk.drbd1938.out0 disk.drbd2118.in0 disk.drbd1356.in0$ memory_percent41.23130440711975 disk.drbd1550.out0 disk.drbd1965.out0! proc.memory.rssMB 11260.40625 network.recv343271 disk.drbd2078.out0 disk.drbd2041.in0 disk.drbd1552.out0 disk.sda1.in0 network.sent 5.012097e+06 disk.drbd1285.out0.2- _wandbruntime2452  _runtime2452i$r ' lr-AdamW/pg10.00014172125348269822 ' lr-AdamW/pg20.00014172125348269822 trainer/global_step5499 % _timestamp1.7857977396452363e+09 _runtime2452.468196295 _step436RX ' lr-AdamW/pg10.00014172125348269822 ' lr-AdamW/pg20.00014172125348269822 trainer/global_step5499 % _timestamp1.7857977396452363e+09 _runtime2452.468196295 _step436x2- _wandbruntime2452  _runtime2452Dh׾'O GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed99260893433344 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed58076869 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated94421739224064 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 4970202947584 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4827327862300 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated37406594 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed92075492437472 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 4827413665424 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 4839258678272 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current246 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated21655897 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated96902923142256 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed94421722184704 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed36541077 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed94421722184704 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed96902820299772 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated36541079 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed37406348 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed21655895 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated94421739224064 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed15750453 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed94617946 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated99260997902336 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated58086525 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4970331524608 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 _runtime2452.470391002 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 _step437 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 trainer/global_step5499 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated99260997902336 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated58086525 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4839171248640 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4839258678272 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 4839171248640 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current244 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed36541077 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated92075509476832E'O N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 _runtime2452.470391002 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 _step437 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 trainer/global_step5499 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated99260997902336 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated58086525 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4839171248640 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4839258678272 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 4839171248640 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current244 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed36541077 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated92075509476832 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed99260893433344 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed58076869 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated94421739224064 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 4970202947584 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4827327862300 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated37406594 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed92075492437472 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 4827413665424 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 4839258678272 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current246 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated21655897 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated96902923142256 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed94421722184704 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed36541077 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed94421722184704 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed96902820299772 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated36541079 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed37406348 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed21655895 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated94421739224064 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed15750453 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed94617946 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated99260997902336 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated58086525 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 4970331524608L2- _wandbruntime2452  _runtime2452Uw M [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 143249920 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed36547720 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated58097062 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed37413165 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed96920437945314 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed92092232609792 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed99633272867328 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4840138305024 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed36547720 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated21659834 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated94662198755328 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated99279044292096 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 4840050868224 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.alylocated94438905987072 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed15753333 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 4971099015680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed94438888947712 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated99633441020928 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed58087392 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated94438905987072 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 4840138305024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated36547722 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4971242265600 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4828205335522 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated36547722 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated94644784 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated15753582 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed99278939815936 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 4840050868224 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated96920540787910 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed94635112 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 168153600 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 4828291138758 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 4828291138758 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed36547720 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated58097062 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed37413165 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed96920437945314 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed92092232609792 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed99633272867328 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4840138305024 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed36547720 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated21659834 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated94662198755328 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated99279044292096 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 4840050868224 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated94438905987072 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed15753333 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 4971099015680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed94438888947712 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated99633441020928 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed58087392 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated94438905987072 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 4840138305024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated36547722 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 4971242265600 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4828205335522 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated36547722 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated94644784 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated15753582 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 O8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed99278939815936 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 4840050868224 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated96920540787910 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed94635112 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548wZ\2- _wandbruntime2452  _runtime2452*` /loss/FBt2/detection_loss1.8744449218114356 epoch0 _step439 /loss/SRqqt2/detection_loss1.10992846544832 2loss/SRqqt2/assignment_loss1.5234989163624804 0loss/FBt1/assignment_loss0.1637546697126993 1loss/FBt2/assignment_loss0.26180128986015916 /loss/FRt2/detection_loss2.7199554509268355 trainer/global_step5499 _runtime 2452.84260192 /loss/FBt1/detection_loss1.2906850848297207 $loss/total_loss21.9989506800955 0loss/FRt1/assignment_loss3.9413255681895247 /loss/FRt2/assignment_loss2.998144200214973 /loss/FRt1/detection_loss2.5174446208308443 1loss/SRqqt1/detection_loss2.0057770614371155 % _timestamp1.7857977400197728e+09 2loss/SRqqt1/assignment_loss1.5921904304713914Q  2loss/SRqqt1/assignment_loss1.5921904304713914 /loss/FBt2/detection_loss1.8744449218114356 epoch0 _step439 /loss/SRqqt2/detection_loss1.10992846544832 2loss/SRqqt2/assignment_loss1.5234989163624804 0loss/FBt1/assignment_loss0.1637546697126993 1loss/FBt2/assignment_loss0.26180128986015916 /loss/FRt2/detection_loss2.7199554509268355 trainer/global_step5499 _runtime 2452.84260192 /loss/FBt1/detection_loss1.2906850848297207 $loss/total_loss21.9989506800955 0loss/FRt1/assignment_loss3.9413255681895247 /loss/FRt2/assignment_loss2.998144200214973 /loss/FRt1/detection_loss2.5174446208308443 1loss/SRqqt1/detection_loss2.0057770614371155 % _timestamp1.7857977400197728e+09.(: U gpu.0.gpu18.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts127.724' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent42.574666666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0bY8Q(:  U disk.drbd1226.in0 disk.drbd2118.out0 disk.sda2.in0 disk.drbd1817.in0 disk.drbd1965.in0 disk.drbd1552.in0( disk./.usagePercent32.3438588449664$ disk./.usageGB578.3410453796387 disk.drbd2078.out0 disk.drbd1584.in0 disk.drbd2041.in0 disk.drbd1356.out0 disk.drbd1673.in0 disk.drbd1550.out0 disk.rbd0.in0 disk.sda1.out0 disk.dm-0.in 2.47265625 disk.dm-0.out101.1201171875 disk.rbd1.in0 disk.drbd1673.out0 disk.drbd1550.in0* proc.memory.availableMB38514.85546875 network.recv344420 disk.sda2.out 66.91015625 disk.drbd1289.out0 proc.cpu.threads29 disk.drbd1510.in0 disk.drbd1510.out0 disk.drbd1226.out0 disk.drbd1356.in0 network.sent 5.017216e+06 disk.sda.in0 disk.drbd1285.in0 disk.drbd1938.out0 disk.drbd1584.out0 disk.drbd1289.in0 cpu13.915481160884747 disk.drbd1931.out0$ memory_percent41.23099446296692 disk.rbd0.out0 disk.sda.out 66.91015625 disk.sda1.in0 disk.drbd1285.out0 disk.drbd2118.in0 disk.drbd1931.in0 disk.drbd2078.in0 disk.nvme0n1.in 2.47265625 disk.rbd1.out9.90625 disk.drbd1965.out0! proc.memory.rssMB 11260.40625 disk.drbd1938.in0# disk.nvme0n1.out101.1201171875 disk.drbd1552.out0 disk.drbd2111.out0 disk.drbd2041.out0 disk.drbd1828.out0 disk.drbd2111.in0 disk.drbd1817.out0 disk.drbd1828.in0* proc.memory.percent17.182016372680664Mؤ2- _wandbruntime2474  _runtime2474 / ' lr-AdamW/pg10.00014300986280696353 ' lr-AdamW/pg20.00014300986280696353 trainer/global_step5549 % _timestamp1.7857977616074314e+09 _runtime2474.430393911 _step440K8 _runtime2474.430393911 _step440 ' lr-AdamW/pg10.00014300986280696353 ' lr-AdamW/pg20.00014300986280696353 trainer/global_step5549 % _timestamp1.7857977616074314e+09AF2- _wandbruntime2474  _runtime2474S'O @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 _step441 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4871216325808 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed100163227719168 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4883167384064 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 5015518643712 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 4871302128932 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated21852747 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed95280060335104 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated15893396 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated100520866122240 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated95505347478528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 % _timestamp1.7857977616083696e+09 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed36873227 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated92912518092832 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 trainer/global_step5549 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5015385872384 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated100163332188160 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated37746143 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated95280077374464 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed21852745 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed97783717379280 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed95478098 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed100163227719168 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4883254813696 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed37745883 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed95280060335104 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated95487756 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed58604871 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed100520708447232 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed100163227719168 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4883254813696 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed37745883 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed95280060335104 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated95487756 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed58604871 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed100520708447232 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4883167384064 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 5015518643712 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 4871302128932 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated21852747 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed95280060335104 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated15893396 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated100520866122240 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated95505347478528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 % _timestamp1.7857977616083696e+09 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed36873227 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated92912518092832 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 trainer/global_step5549 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5015385872384 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated100163332188160 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated37746143 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated95280077374464 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed21852745 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed97783717379280 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed95478098 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated1216 2- _wandbruntime2474  _runtime2474@kJ&M KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current248 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 4884047003648 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated37752880 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed37752630 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated36879872 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated95504936 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 trainer/global_step5549 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated36879872 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4884134440448 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed95522529691648 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed97801335024822 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed36879870 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed100538827868672 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated58625064 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated95297244137472 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 4884134440448 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated100538970856448 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed95495264 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated95522554595328 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated95504936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4872093799030 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 % _timestamp1.7857977619720633e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated21856684 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated15896196 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed95297227098112 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed58615394 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed95297227098112 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated100181378577920 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed100181274101760 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed21856682 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed36879870 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated100181378577920 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current250 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 142987776 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated95297244137472 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 118084096 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed95495264 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291a&M ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated95297244137472 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 4884134440448 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated100538970856448 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed95495264 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated95522554595328 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated95504936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4872093799030 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 % _timestamp1.7857977619720633e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated21856684 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated15896196 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed95297227098112 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed58615394 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed95297227098112 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated100181378577920 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed100181274101760 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed21856682 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed36879870 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated100181378577920 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current250 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 142987776 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated95297244137472 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 118084096 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed95495264 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current248 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 4884047003648 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated37752880 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed37752630 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated36879872 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated95504936 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 trainer/global_step5549 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated36879872 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4884134440448 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed95522529691648 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed97801335024822 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed36879870 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed100538827868672 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated58625064 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0Ze2- _wandbruntime2474  _runtime2474& 1loss/SRqqt1/detection_loss2.3077494232427505 trainer/global_step5549 _runtime2474.796500716 _step443 2loss/SRqqt1/assignment_loss1.4088599795386905 /loss/FRt1/detection_loss2.5099836449874076 1loss/SRqqt2/detection_loss0.9335447854938961 % _timestamp1.7857977619736614e+09 .loss/FRt1/assignment_loss4.04929956574189 2loss/SRqqt2/assignment_loss1.6044920038077093 /loss/FBt1/detection_loss0.9666991811542911 /loss/FBt2/detection_loss1.3734707783679312 %loss/total_loss21.18632252554519 epoch0 1loss/FBt1/assignment_loss0.03479655475902209 1loss/FBt2/assignment_loss0.47504185325647086 .loss/FRt2/detection_loss2.442847660609654 /loss/FRt2/assignment_loss3.079537094585479$ /loss/FBt1/detection_loss0.9666991811542911 /loss/FBt2/detection_loss1.3734707783679312 %loss/total_loss21.18632252554519 epoch0 1loss/FBt1/assignment_loss0.03479655475902209 1loss/FBt2/assignment_loss0.47504185325647086 .loss/FRt2/detection_loss2.442847660609654 /loss/FRt2/assignment_loss3.079537094585479 1loss/SRqqt1/detection_loss2.3077494232427505 trainer/global_step5549 _runtime2474.796500716 _step443 2loss/SRqqt1/assignment_loss1.4088599795386905 /loss/FRt1/detection_loss2.5099836449874076 1loss/SRqqt2/detection_loss0.9335447854938961 % _timestamp1.7857977619736614e+09 .loss/FRt1/assignment_loss4.04929956574189 2loss/SRqqt2/assignment_loss1.6044920038077093n(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts125.097' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent41.699 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0ET(:  U disk.drbd1938.in0 disk.drbd1552.in0 disk.drbd1552.out0 disk.drbd1584.in0 disk.drbd1584.out0 disk.drbd1285.in0 disk.drbd1285.out0 disk.drbd2111.out0 disk.rbd1.in0$ disk./.usageGB578.3412399291992 disk.dm-0.in 2.47265625 disk.drbd2078.out0 disk.nvme0n1.in 2.47265625 cpu13.913523060577303 network.recv346825 disk.rbd0.in0 disk.drbd2111.in0 disk.rbd1.out9.96875 disk.drbd1938.out0 disk.drbd1510.in0 disk.dm-0.out101.5654296875 disk.drbd1226.out0 disk.drbd2118.in0 network.sent 5.062261e+06 disk.drbd1931.out0 disk.sda1.out0 disk.drbd2078.in0 disk.drbd1356.in0 proc.cpu.threads29 disk.drbd1673.out0* disk./.usagePercent32.343869725196285 disk.drbd2041.out0 disk.drbd1550.in0 disk.drbd1289.in0 disk.rbd0.out0# disk.nvme0n1.out101.5654296875 disk.drbd1356.out0% memory_percent41.232842206954956* proc.memory.percent17.182016372680664 disk.drbd1550.out0 disk.sda.in0 disk.sda1.in0 disk.drbd1817.in0 disk.drbd1931.in0 disk.drbd2041.in0 disk.sda.out 67.44921875 disk.drbd2118.out0 disk.drbd1817.out0 disk.drbd1673.in0 disk.drbd1226.in0 disk.sda2.out 67.44921875* proc.memory.availableMB38513.64453125 disk.drbd1828.in0 disk.drbd1828.out0! proc.memory.rssMB 11260.40625 disk.drbd1510.out0 disk.sda2.in0 disk.drbd1965.in0 disk.drbd1965.out0 disk.drbd1289.out0!(: мU gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts125.459' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent41.81966666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0%I(:  V disk.dm-0.in 2.47265625 disk.dm-0.out101.8115234375 disk.drbd1817.in0 disk.drbd1965.in0 disk.drbd1965.out0 disk.drbd1552.out0 disk.rbd0.out0 disk.drbd1285.in0 cpu13.911042443616262 disk.drbd1550.in0 disk.drbd1550.out0 disk.drbd2078.out0 disk.drbd1584.out0 disk.drbd1356.out0 disk.sda2.in0 disk.sda1.out0 disk.drbd1285.out0 disk.drbd1510.in0 disk.drbd1226.out0* proc.memory.percent17.182016372680664$ disk./.usageGB578.3412704467773 di^Usk.drbd1510.out0 disk.drbd2078.in0 disk.rbd1.out9.96875 network.sent 5.107288e+06 disk.drbd1673.in0 disk.drbd1828.out0 disk.drbd1289.out0 disk.drbd1938.out0 disk.drbd1584.in0! proc.memory.rssMB 11260.40625 disk.drbd1552.in0) proc.memory.availableMB 38514.3359375 network.recv348871) disk./.usagePercent32.34387143189901 disk.drbd2041.in0 disk.drbd1828.in0 disk.drbd2111.out0 disk.drbd1931.out0 disk.drbd1289.in0 disk.drbd2111.in0 disk.drbd1938.in0 disk.rbd0.in0 disk.drbd2118.in0 disk.drbd2118.out0 disk.drbd1817.out0$ memory_percent41.23178720474243 disk.drbd2041.out0 disk.drbd1226.in0 disk.rbd1.in0 disk.sda.out 67.7109375 proc.cpu.threads29 disk.drbd1673.out0 disk.nvme0n1.in 2.47265625# disk.nvme0n1.out101.8115234375 disk.drbd1356.in0 disk.sda2.out 67.7109375 disk.drbd1931.in0 disk.sda1.in0 disk.sda.in02h2- _wandbruntime2496  _runtime2496QK _step444 ' lr-AdamW/pg10.00014429847213122884 ' lr-AdamW/pg20.00014429847213122884 trainer/global_step5599 % _timestamp1.7857977839161346e+09 _runtime2496.739086728.  _runtime2496.739086728 _step444 ' lr-AdamW/pg10.00014429847213122884 ' lr-AdamW/pg20.00014429847213122884 trainer/global_step5599 % _timestamp1.7857977839161346e+092<2- _wandbruntime2496  _runtime2496'O _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated96138415524864 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated16037515 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated96347908 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed96138398485504 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 _runtime 2496.74030885 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 % _timestamp1.7857977839170508e+09 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 4915190592440 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current242 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 _step445 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated22049597 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4915104789316 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 trainer/global_step5599 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated59142529 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed59132873 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated38087112 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed101065562004992 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4927250949120 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated101426392893952 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed96365678414848 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated101065666473984 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated96365703318528 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4927163519488 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed101065562004992 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4927250949120 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated101426392893952 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed96365678414848 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated101065666473984 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated96365703318528 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4927163519488 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated96347908 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed96138398485504 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 _runtime 2496.74030885 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 % _timestamp1.7857977839170508e+09 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 4915190592440 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current242 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 _step445 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated22049597 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4915104789316 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 trainer/global_step55992- _wandbruntime2497  _runtime2497,U%&M A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated101083712863744 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated59153066 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed96155565248512 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated96155582287872 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated101444506090496 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated96365088 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated96365088 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated37212022 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated93766266881152 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5061595655168 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed38093603 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 4928130575872 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated59153066 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated101083712863744 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 5061456599552 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 _runtime2497.107971745 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed22053532 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed96155565248512 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4915982262538 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4928130575872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 _step446 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 4928043139072 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed98682232104330 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 163959296 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 4928043139072 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed93766249841792 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed37212020 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated37212022 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed96355416 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed101444342131200 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated98682334946926 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 P.M JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated37212022 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated93766266881152 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5061595655168 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_ #pool.peak11816 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed38093603 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 4928130575872 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated59153066 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated101083712863744 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 187192320 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 5061456599552 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 _runtime2497.107971745 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed22053532 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed96155565248512 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4915982262538 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4928130575872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 _step446 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 4928043139072 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed98682232104330 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 163959296 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 4928043139072 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed93766249841792 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed37212020 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated37212022 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed96355416 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed101444342131200 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated98682334946926 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated96155582287872 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated101444506090496 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated96365088 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated96365088A82- _wandbruntime2497  _runtime2497Э 1loss/FBt1/assignment_loss0.05074875798548646 0loss/FBt2/assignment_loss0.0465789941517869 0loss/SRqqt1/detection_loss2.142204328422883 1loss/SRqqt2/detection_loss1.0350091475418504 % _timestamp1.7857977842863035e+09 /loss/FRt1/assignment_loss4.041179232078023 2loss/SRqqt2/assignment_loss1.3535923652442885 /loss/FRt2/detection_loss2.9786152451609573 epoch0 2loss/SRqqt1/assignment_loss1.6086371353379003 /loss/FRt1/detection_loss2.3924004984374094 /loss/FBt1/detection_loss1.4677945011574776 _step447 0loss/FRt2/assignment_loss2.4943699141343436 .loss/FBt2/detection_loss7.338346984004602 %loss/total_loss26.94947710365701 trainer/global_step5599 _runtime2497.109125262g 0loss/FBt2/assignment_loss0.0465789941517869 0loss/SRqqt1/detection_loss2.142204328422883 1loss/SRqqt2/detection_loss1.0350091475418504 % _timestamp1.7857977842863035e+09 /loss/FRt1/assignment_loss4.041179232078023 2loss/SRqqt2/assignment_loss1.3535923652442885 /loss/FRt2/detection_loss2.9786152451609573 epoch0 2loss/SRqqt1/assignment_loss1.6086371353379003 /loss/FRt1/detection_loss2.3924004984374094 /loss/FBt1/detection_loss1.4677945011574776 _step447 0loss/FRt2/assignment_loss2.4943699141343436 .loss/FBt2/detection_loss7.338346984004602 %loss/total_loss26.94947710365701 trainer/global_step5599 _runtime2497.109125262 1loss/FBt1/assignment_loss0.05074875798548646Db(: ٱU gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp41.0 gpu.0.powerWatts97.521' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent32.507000000000005 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0S X(:  U disk.sda2.in0 disk.drbd2111.out0 disk.drbd1550.in0 disk.drbd1226.out0 disk.sda2.out 68.06640625 disk.drbd1965.in0$ memory_percent41.23227000236511 disk.drbd2078.out0 cpu13.909277476792644 disk.sda.in0# disk.nvme0n1.out102.3115234375 disk.drbd1356.in0 disk.drbd1673.in0 disk.dm-0.out102.3115234375 disk.drbd2111.in0 disk.drbd1828.in0 network.recv348937* proc.memory.availableMB38514.01953125 disk.sda.out 68.06640625 disk.sda1.in0 disk.drbd1510.out0 disk.drbd1817.out0* proc.memory.percent17.182016372680664 disk.rbd0.in0 disk.drbd2118.in0 disk.nvme0n1.in 2.47265625 disk.drbd2041.out0 disk.drbd1552.out0 proc.cpu.threads29 disk.drbd1510.in0 disk.drbd1356.out0 disk.drbd1673.out0 disk.drbd1289.in0 disk.rbd1.out 10.0009765625 disk.drbd1931.in0 disk.drbd2041.in0! proc.memory.rssMB 11260.40625 network.sent 5.107354e+06 disk.drbd1584.in0 disk.drbd1828.out0 disk.drbd1938.out0$ disk./.usageGB578.3412895202637 disk.drbd1285.in0 disk.dm-0.in 2.47265625 disk.drbd2118.out0) disk./.usagePercent32.34387249858822 disk.rbd0.out0 disk.sda1.out0 disk.drbd1817.in0 disk.rbd1.in0 disk.drbd1226.in0 disk.drbd2078.in0 disk.drbd1965.out0 disk.drbd1584.out0 disk.drbd1552.in0 disk.drbd1550.out0 disk.drbd1938.in0 disk.drbd1285.out0 disk.drbd1931.out0 disk.drbd1289.out0w8r+2- _wandbruntime2518  _runtime2518W ' lr-AdamW/pg10.00014558708145549415 ' lr-AdamW/pg20.00014558708145549415 trainer/global_step5649 % _timestamp1.7857978053006206e+09 _runtime2518.123598384 _step448P#! _runtime2518.123598384 _step448 ' lr-AdamW/pg10.00014558708145549415 ' lr-AdamW/pg20.00014558708145549415 trainer/global_step5649 % _timestamp1.7857978053006206e+09>U2- _wandbruntime2518  _runtime2518ٙ,'O _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated96996753675264 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 4971159654912 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated97208060 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed37537527 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed59660875 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4958993252824 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated94586535324832 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated101968000759808 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed101967896290816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated97226059158528 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed99545511538296 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4971159654912 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4971247084544 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated37537529 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5105717483008 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed101967896290816 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated38426949 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed37537527 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed94586518285472 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 4959079055948 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392q=$O GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed37537527 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed94586518285472 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 4959079055948 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated96996753675264 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 4971159654912 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated97208060 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed37537527 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed59660875 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 4958993252824 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated94586535324832 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated101968000759808 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed101967896290816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated97226059158528 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed99545511538296 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 4971159654912 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 4971247084544 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated37537529 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5105717483008 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed101967896290816 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated38426949 TO2- _wandbruntime2518  _runtime2518&M REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed16183093 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 4959870726046 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed22250382 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current241 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed37544170 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated59681068 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated102350014925312 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated37544172 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed102349865646080 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated97243266275328 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _runtime2518.481771291 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed97215568 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated101986047149568 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed97013903398912 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated97225240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated101986047149568 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed38433475 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5106748649984 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated16183332 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4972126711296 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated38433716 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated99563232026434 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed101985942673408 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated99563232026434 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed101985942673408 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated37544172 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed102349865646080 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated97243266275328 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _runtime2518.481771291 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed97215568 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated101986047149568 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed97013903398912 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated97225240 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated101986047149568 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed38433475 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5106748649984 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated16183332 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 4972126711296 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated38433716 EDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed102870230576640 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed98086390094848 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated97855091825664 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed37869677 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 % _timestamp1.7857978273338404e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed22443295 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed38767172 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed103237310007808 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated22443297 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658A'O TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed22443295 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed38767172 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed103237310007808 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated22443297 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated37869679 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5015243219968 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated98068212 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 _runtime2540.157129081 _step453 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5002881716332 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated102870335045632 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed100426408617804 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current242 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated100426511460288 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 5015155790336 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed16323877 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed37869677 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed97855074786304 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed95423526901472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated97855091825664 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated60198533 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current240 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 187192320 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed97855074786304 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed60188877 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed102870230576640 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed98086390094848 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated97855091825664 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed37869677 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 % _timestamp1.7857978273338404e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 27682406402- _wandbruntime2540  _runtime2540Q&M C6DeviceStatsMonitor.on_train_batch_end/active.all.freed98075720 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed60199400 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 5151819765760 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed100444026263346 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed97872241549312 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed60199400 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated37876322 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated16326950 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated97872258588672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated97872258588672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated98085392 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated98103622115328 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed95440267073792 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5016122846720 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed22447232 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed98103597211648 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed98075720 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current262 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated22447234 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed102888276959232 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated100444129105942 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 5016035409920 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 _runtime2540.523377005 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5016122846720 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed38773922 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 % _timestamp1.7857978277000444e+09 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated103255576742400 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated60209070 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed97872241549312 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated98085392 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated102888381435392 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current260 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated95440284113152 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated102888381435392 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current260 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated95440284113152 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed60199400 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated37876322 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated16326950 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated97872258588672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated97872258588672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated98085392 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated98103622115328 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed95440267073792 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.aI44llocated 5016122846720 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed22447232 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed98103597211648 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed98075720 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current262 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated22447234 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed102888276959232 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated100444129105942 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 5016035409920 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 _runtime2540.523377005 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5016122846720 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed38773922 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 % _timestamp1.7857978277000444e+09.2- _wandbruntime2540  _runtime2540 epoch0 _step455 2loss/SRqqt1/assignment_loss1.7253184548833154 1loss/SRqqt2/detection_loss1.1038007466068673 &loss/total_loss23.012806327461576 0loss/FRt2/assignment_loss2.9868597539988433 2loss/SRqqt2/assignment_loss1.4960154659098628 0loss/FBt2/assignment_loss0.2653931703728934 1loss/SRqqt1/detection_loss2.2094216637990693 /loss/FBt1/detection_loss1.5777150644622593 /loss/FBt2/detection_loss2.6006739139556885 trainer/global_step5699 % _timestamp1.7857978277016993e+09 /loss/FRt1/assignment_loss3.860728744987969 1loss/FBt1/assignment_loss0.13250448056791916 /loss/FRt1/detection_loss2.3248744939898587 _runtime 2540.52452962 /loss/FRt2/detection_loss2.7295003739270296<  2loss/SRqqt1/assignment_loss1.7253184548833154 1loss/SRqqt2/detection_loss1.1038007466068673 &loss/total_loss23.012806327461576 0loss/FRt2/assignment_loss2.9868597539988433 2loss/SRqqt2/assignment_loss1.4960154659098628 0loss/FBt2/assignment_loss0.2653931703728934 1loss/SRqqt1/detection_loss2.2094216637990693 /loss/FBt1/detection_loss1.5777150644622593 /loss/FBt2/detection_loss2.6006739139556885 trainer/global_step5699 % _timestamp1.7857978277016993e+09 /loss/FRt1/assignment_loss3.860728744987969 1loss/FBt1/assignment_loss0.13250448056791916 /loss/FRt1/detection_loss2.3248744939898587 _runtime 2540.52452962 /loss/FRt2/detection_loss2.7295003739270296 epoch0 _step455^N(: ͹U gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp40.0 gpu.0.powerWatts109.424' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent36.474666666666664 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0I@T(:  ͹ߣU disk.drbd1289.in0 disk.rbd0.in0 disk.sda2.in0& proc.memory.availableMB 38514.0625 disk.drbd1550.out0 proc.cpu.threads29 network.sent 5.155555e+06 disk.drbd2041.in0 disk.drbd1285.in0 disk.drbd1285.out0 disk.drbd2118.out0 disk.rbd1.in0 disk.drbd1965.in0 disk.drbd1673.in0 disk.drbd1552.out0 disk.dm-0.out103.1220703125 disk.drbd1226.out0 disk.drbd1584.out0 disk.drbd1965.out0 disk.drbd2118.in0 disk.drbd1510.out0* proc.memory.percent17.182016372680664 disk.drbd1226.in0 disk.drbd1828.out0 disk.drbd1938.out0 disk.sda.in0 disk.drbd1931.out0 disk.drbd1510.in0 disk.drbd2078.out0 disk.sda2.out 69.08984375! proc.memory.rssMB 11260.40625) disk./.usagePercent32.34387143189901 disk.drbd1938.in0 disk.dm-0.in 2.47265625# disk.nvme0n1.out103.1220703125 disk.drbd1356.in0 disk.rbd0.out0 disk.sda1.out0 disk.drbd1584.in0 disk.drbd1817.out0 disk.rbd1.out 10.1572265625 disk.drbd1828.in0 disk.sda.out 69.08984375$ memory_percent41.23220443725586 disk.drbd1673.out0 disk.drbd1931.in0 disk.drbd2041.out0 disk.drbd1289.out0 disk.drbd2078.in0 disk.nvme0n1.in 2.47265625 disk.drbd2111.out0 cpu13.903580537213044$ disk./.usageGB578.3412704467773 disk.drbd1550.in0 disk.drbd2111.in0 disk.drbd1817.in0 network.recv352339 disk.drbd1552.in0 disk.sda1.in0 disk.drbd1356.out0#2- _wandbruntime2562  _runtime2562J] ' lr-AdamW/pg20.00014816430010402475 trainer/global_step5749 $ _timestamp1.785797849185251e+09 _runtime2562.008186976 _step456 ' lr-AdamW/pg10.00014816430010402475!X trainer/global_step5749 $ _timestamp1.785797849185251e+09 _runtime2562.008186976 _step456 ' lr-AdamW/pg10.00014816430010402475 ' lr-AdamW/pg20.000148164300104024752- _wandbruntime2562  _runtime2562}9'O WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed60716879 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed16467259 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed96260535517472 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated60726535 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5046770179840 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated38201829 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed60716879 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current262 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed98946745934848 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated16467519 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated22640147 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated60726535 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 5059151925760 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 5046855982964 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5059239355392 % _timestamp1.7857978491862228e+09 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed104142816980992 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5196071046144 _runtime2562.009497398 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated98928364 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 _step457 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed38201827 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated38201829 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 5196208011776 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed98918706 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 trainer/global_step5749 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed98713412936704 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated96260552556832 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed22640145 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated98946770838528 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed103772564862464 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5059239355392 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed101307305697312 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856A('O L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed38201827 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated38201829 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 5196208011776 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed98918706 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 trainer/global_step5749 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed98713412936704 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated96260552556832 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed22640145 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated98946770838528 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed103772564862464 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5059239355392 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed101307305697312 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed60716879 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed16467259 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed96260535517472 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated60726535 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5046770179840 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated38201829 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed60716879 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current262 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed98946745934848 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated16467519 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated22640147 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated60726535 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 5059151925760 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 5046855982964 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5059239355392 % _timestamp1.7857978491862228e+09 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed104142816980992 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5196071046144 _runtime2562.009497398 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated98928364 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 _step457b2- _wandbruntime2562  _runtime25626 M A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated98945544 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 5196973798912 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed38208470 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated103790715721216 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed101324923342854 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated103790715721216 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 % _timestamp1.7857978495633466e+09 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated38208472 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated22644084 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5197110757376 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated96277292729152 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current264 GADevice(xnStatsMonitor.on_train_batch_end/oversize_segments.allocated0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 5047647653062 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 _runtime2562.386694636 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed39114181 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed60727402 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed98730579699712 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5060118982144 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current262 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 trainer/global_step5749 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0&M QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current264 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 5047647653062 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 _runtime2562.386694636 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed39114181 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed60727402 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed98730579699712 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5060118982144 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current262 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 trainer/global_step5749 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated98945544 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 5196973798912 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed38208470 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated103790715721216 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed101324923342854 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated103790715721216 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 % _timestamp1.7857978495633466e+09 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated38208472 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated22644084 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5197110757376 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated96277292729152Iq2- _wandbruntime2562  _runtime2562tb /loss/FRt2/assignment_loss2.766055087921983 .loss/FRt2/detection_loss2.487266720035824 1loss/FBt2/assignment_loss0.15003251437246856 /loss/FRt1/detection_loss2.5168196158383482 /loss/FBt1/detection_loss1.0116419142455255 /loss/FBt2/detection_loss2.1469670683145523 /loss/FRt1/assignment_loss3.950561149145967 1loss/FBt1/assignment_loss0.14377296394251826 1loss/SRqqt2/detection_loss1.0927728003977488 trainer/global_step5749 % _timestamp1.7857978495650017e+09 epoch0 2loss/SRqqt1/assignment_loss1.7083919048309326 1loss/SRqqt2/assignment_loss1.457332771326903 1loss/SRqqt1/detection_loss2.4582944687675026 &loss/total_loss21.889908979140273 _runtime2562.387830188 _step459Bx  /loss/FBt1/detection_loss1.0116419142455255 /loss/FBt2/detection_loss2.1469670683145523 /loss/FRt1/assignment_loss3.950561149145967 1loss/FBt1/assignment_loss0.14377296394251826 1loss/SRqqt2/detection_loss1.0927728003977488 trainer/global_step5749 % _timestamp1.7857978495650017e+09 epoch0 2loss/SRqqt1/assignment_loss1.7083919048309326 1loss/SRqqt2/assignment_loss1.457332771326903 1loss/SRqqt1/detection_loss2.4582944687675026 &loss/total_loss21.889908979140273 _runtime2562.387830188 _step459 /loss/FRt2/assignment_loss2.766055087921983 .loss/FRt2/detection_loss2.487266720035824 1loss/FBt2/assignment_loss0.15003251437246856 /loss/FRt1/detection_loss2.5168196158383482(+(: ܹU gpu.0.gpu30.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp40.0 gpu.0.powerWatts97.137' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent32.379000000000005 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0L5Q(:  ܹU disk.drbd1828.in0 disk.drbd2111.out0 network.recv354711 disk.sda2.out 69.515625 disk.sda1.in0 disk.rbd1.out 10.189453125 disk.drbd1673.out0 disk.drbd1285.in0 disk.drbd1584.out0 disk.drbd1931.out0 disk.drbd1550.out0 disk.sda1.out0 disk.drbd1285.out0 disk.drbd1584.in0 disk.drbd2118.out0 disk.drbd1356.out0 disk.drbd1828.out0 disk.drbd1550.in0 disk.rbd0.out0 disk.drbd2118.in0 disk.dm-0.in 2.47265625 disk.drbd2078.in0 disk.sda.in0$ memory_percent41.23300313949585) proc.memory.availableMB 38513.5390625 disk.drbd1552.out0 disk.drbd1965.in0 disk.drbd1938.in0 disk.drbd1510.in0 disk.drbd1226.out0* proc.memory.percent17.182016372680664 disk.drbd1931.in0 disk.drbd1289.out0 disk.rbd0.in0 disk.drbd1226.in0 disk.drbd2078.out0 cpu13.90170368486044# disk.nvme0n1.out103.2822265625 disk.sda2.in0 disk.drbd1817.in0 proc.cpu.threads29 disk.drbd1510.out0 disk.dm-0.out103.2822265625$ disk./.usageGB578.3412895202637 disk.drbd1817.out0 disk.drbd1552.in0 disk.drbd1938.out0 disk.sda.out 69.515625! proc.memory.rssMB 11260.40625 disk.drbd1673.in0 disk.drbd2111.in0 network.sent 5.200593e+06 disk.drbd1356.in0 disk.rbd1.in0 disk.drbd1289.in0 disk.drbd1965.out0 disk.drbd2041.out0) disk./.usagePercent32.34387249858822 disk.nvme0n1.in 2.47265625 disk.drbd2041.in0pU(:  U disk.drbd1552.in0 disk.drbd1289.out0 disk.drbd1226.out0 disk.drbd1285.in0 disk.drbd1584.in0# disk.nvme0n1.out103.5166015625 disk.drbd1356.in0 disk.drbd1356.out0 disk.sda.in0 disk.drbd1226.in0 disk.drbd1965.out0! proc.memory.rssMB 11260.40625 disk.drbd1285.out0 disk.drbd1510.out0 disk.nvme0n1.in 2.47265625 disk.rbd1.out 10.283203125 disk.drbd1965.in0) disk./.usagePercent32.34387143189901 network.recv356623 disk.drbd1673.out0 disk.drbd2078.in0 disk.sda1.in0 disk.drbd2118.in0 disk.sda2.in0 disk.drbd1931.out0 disk.drbd2041.out0 disk.drbd1828.out0 disk.sda1.out0* proc.memory.availableMB38514.28515625$ memory_percent41.23186469078064 disk.drbd1673.in0 disk.drbd1510.in0 disk.drbd2111.in0$ disk./.usageGB578.3412704467773 disk.dm-0.in 2.47265625 cpu13.899902851823459 disk.drbd2041.in0 disk.drbd1552.out0 disk.drbd1584.out0 disk.sda2.out 69.7421875 disk.rbd1.in0 network.sent 5.248399e+06 disk.drbd1828.in0 disk.drbd2078.out0 disk.drbd1817.in0* proc.memory.percent17.182016372680664 proc.cpu.threads29 disk.drbd1938.out0 disk.rbd0.out0 disk.drbd1931.in0 disk.drbd1550.in0 disk.drbd1550.out0 disk.drbd1289.in0 disk.drbd1938.in0 disk.drbd1817.out0 disk.dm-0.out103.5166015625 disk.drbd2118.out0 disk.drbd2111.out0 disk.rbd0.in0 disk.sda.out 69.7421875M:(: U gpu.0.gpu23.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp40.0 gpu.0.powerWatts124.429' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent41.47633333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0 $]2- _wandbruntime2583  _runtime2583Y@ ' lr-AdamW/pg10.00014945290942829006 ' lr-AdamW/pg20.00014945290942829006 trainer/global_step5799 $ _timestamp1.785797871154355e+09 _runtime 2583.97729518 _step460TB| _runtime 2583.97729518 _step460 ' lr-AdamW/pg10.00014945290942829006 ' lr-AdamW/pg20.00014945290942829006 trainer/global_step5799 $ _timestamp1.785797871154355e+092h#2- _wandbruntime2583  _runtime2583B'O E8DeviceStatsMonitor.on_train_batch_start/active.all.freed99778858 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed39447620 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed16610625 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 trainer/global_step5799 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed104674899148288 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed105048352594944 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5090658643348 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed99571751087104 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated104675003617280 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5103235490816 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated97097561172832 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5103148061184 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed61244881 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated99788516 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated16610877 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current254 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5241250820096 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated38533979 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current252 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated61254537 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated104675003617280 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5103235490816 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated99807126678528 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed97097544133472 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5090658643348 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed99571751087104 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated104675003617280 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5103235490816 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated97097561172832 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5103148061184 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed61244881 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated99788516 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated16610877 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current254 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5241250820096 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated38533979 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current252 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated61254537 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated104675003617280 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5103235490816 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated99807126678528 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed97097544133472 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 trainer/global_step5799 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed104674899148288 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed105048352594944 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 I6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated99805696 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed38540620 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated99805696 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed99796024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated102205923264958 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 % _timestamp1.7857978715218027e+09 _runtime2584.345120549 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 5104027680768 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current246 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated97114301345152 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5091621919806 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated104693050007040 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed97114284305792 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed38540620 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed105066463390208 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated38540622 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 _step462 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5104115117568 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated104693050007040 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5104027680768 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated61265074 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed22840932 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 trainer/global_step5799 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5104115117568 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated39454546 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed102205820422362 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed99588917850112 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current248 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed39454298Lb`&M F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated99805696 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed38540620 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated99805696 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed99796024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated102205923264958 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 % _timestamp1.7857978715218027e+09 _runtime2584.345120549 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 5104027680768 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current246 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated97114301345152 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5091621919806 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated104693050007040 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed97114284305792 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed38540620 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed105066463390208 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated38540622 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 _step462 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5104115117568 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated104693050007040 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5104027680768 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated61265074 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed22840932 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 trainer/global_step5799 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5104115117568 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated39454546 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed102205820422362 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed99588917850112 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current248 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed39454298 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856#z2- _wandbruntime2584  _runtime25842r 1loss/SRqqt2/assignment_loss1.523128936695453 0loss/SRqqt1/detection_loss2.133159379033666 &loss/total_loss21.649206410997024 % _timestamp1.7857978715234673e+09 0loss/FBt2/assignment_loss0.2306638764315291 /loss/FBt2/detection_loss2.0320215757404054 2loss/SRqqt1/assignment_loss1.1500923484563828 /loss/FRt2/detection_loss2.5677367766020875 1loss/SRqqt2/detection_loss1.0443694028981338 0loss/FBt1/assignment_loss0.0713545434166507 .loss/FRt1/detection_loss2.445685382356349 /loss/FBt1/detection_loss1.0492926078640419 epoch0 trainer/global_step5799 _runtime2584.346296116 _step463 /loss/FRt1/assignment_loss3.852396110898441 /loss/FRt2/assignment_loss3.549305470603885 0loss/FBt1/assignment_loss0.0713545434166507 .loss/FRt1/detection_loss2.445685382356349 /loss/FBt1/detection_loss1.0492926078640419 epoch0 trainer/global_step5799 _runtime2584.346296116 _step463 /loss/FRt1/assignment_loss3.852396110898441 /loss/FRt2/assignment_loss3.549305470603885 1loss/SRqqt2/assignment_loss1.523128936695453 0loss/SRqqt1/detection_loss2.133159379033666 &loss/total_loss21.649206410997024 % _timestamp1.7857978715234673e+09 0loss/FBt2/assignment_loss0.2306638764315291 /loss/FBt2/detection_loss2.0320215757404054 2loss/SRqqt1/assignment_loss1.1500923484563828 /loss/FRt2/detection_loss2.5677367766020875 1loss/SRqqt2/detection_loss1.0443694028981338`(: U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp40.0 gpu.0.powerWatts124.09' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent41.36333333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0YJR(:  U disk.drbd1226.out0 disk.drbd2118.out0 disk.drbd1931.out0 disk.drbd2041.in0 disk.drbd1289.in0 disk.drbd1289.out0* proc.memory.percent17.182016372680664$ disk./.usageGB578.3412132263184 disk.drbd2078.in0# disk.nvme0n1.out103.8037109375 disk.rbd1.out 10.345703125 disk.drbd1673.out0! proc.memory.rssMB 11260.40625 proc.cpu.threads29 disk.drbd1584.in0 disk.drbd1817.in0 disk.sda.out70.40625 disk.sda2.in0 disk.drbd1817.out0 disk.dm-0.out103.8037109375 disk.drbd1584.out0 disk.nvme0n1.in 2.47265625 disk.drbd1552.out0 disk.drbd1510.out0 disk.rbd1.in0 disk.drbd1550.in0 disk.sda1.out0 disk.drbd1965.out0 disk.rbd0.in0* disk./.usagePercent32.343868231831394 disk.drbd2118.in0 disk.drbd1673.in0 disk.drbd2041.out0 disk.drbd1828.in0 cpu13.898114452605485 disk.drbd1510.in0 disk.dm-0.in 2.47265625 disk.drbd1356.out0 disk.drbd2111.in0 disk.drbd1965.in0 disk.drbd1285.in0 disk.rbd0.out0* proc.memory.availableMB38513.55859375 disk.drbd2078.out0 disk.drbd1356.in0 disk.sda.in0 disk.sda1.in0 network.recv356689 disk.drbd1226.in0 disk.drbd1550.out0 network.sent 5.248465e+06 disk.drbd1552.in0 disk.drbd1828.out0 disk.drbd1938.out0 disk.drbd1285.out0 disk.sda2.out70.40625 disk.drbd2111.out0 disk.drbd1931.in0 disk.drbd1938.in0$ memory_percent41.23297333717346i2R72- _wandbruntime2606  _runtime26067 ' lr-AdamW/pg10.00015074151875255537 ' lr-AdamW/pg20.00015074151875255537 trainer/global_step5849 % _timestamp1.7857978932968497e+09 _runtime2606.121702629 _step464Hs ' lr-AdamW/pg10.00015074151875255537 ' lr-AdamW/pg20.00015074151875255537 trainer/global_step5849 % _timestamp1.7857978932968497e+09 _runtime2606.121702629 _step4642- _wandbruntime2606  _runtime2606"O UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed105577233434112 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed38866127 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 _step465 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated38866129 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 100639010 NEDeviceStatsMonitor.on_train_batch_start/allo6f#cation.small_pool.current9656 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed16752251 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated105954031641088 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed105953880257536 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated97934569788832 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5147231626240 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed100667457614848 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated61782539 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated105577337903104 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated61782539 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated16752488 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated105577337903104 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated39786335 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5286422642688 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated100667482518528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated100430106276864 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5134547106856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated103069202698812 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 trainer/global_step5849 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed105577233434112 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 5147144196608 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current237 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed100430089237504 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current239 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed39786096 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed100430089237504 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 126479872 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed38866127 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed61772883 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed97934552749472 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated38866129 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated23033847 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 5286549122560 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _runtime 2606.12179837 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5147231626240 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed103069099856328 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 $ _timestamp1.785797893297562e+09 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 100648668 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated16752488 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated105577337903104 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated39786335 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5286422642688 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated100667482518528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated100430106276864 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5134547106856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated103069202698812 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 trainer/global_step5849 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed105577233434112 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 5147144196608 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current237 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed100430089237504 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current239 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed39786096 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed100430089237504 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 126479872 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed38866127 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed61772883 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed97934552749472 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated38866129 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated23033847 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 5286549122560 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _runtime 2606.12179837 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5147231626240 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed103069099856328 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 $ _timestamp1.785797893297562e+09 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 100648668 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 100665848 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed105595279816704 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed38872770 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated39793036 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed105971992470016 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated38872772 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 5135424580078 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated105972145943552 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed16754992 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed100447256000512 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5135510383314 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5287456308224 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5148111252992 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated23037784 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed103086717501870 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed38872770 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed23037782 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated105595384292864 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5148023816192 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current262 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated16755252 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated100447273039872 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 _step466 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated97951309961152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 % _timestamp1.7857978936699936e+09 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed100447256000512 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed61783406 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 5287327738368 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated103086820344466 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 100656176 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated61793076 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated100684689635328 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 100656176 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated61793076 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated100447273039872 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed39792774 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 _runtime2606.493335004{H M O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5148111252992 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated23037784 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed103086717501870 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed38872770 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed23037782 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated105595384292864 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5148023816192 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current262 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated16755252 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated100447273039872 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 _step466 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated97951309961152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 % _timestamp1.7857978936699936e+09 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed100447256000512 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed61783406 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 5287327738368 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated103086820344466 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 100656176 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated61793076 MGDeviceStatsMonitor.on_train_Hbatch_end/inactive_split.large_pool.current2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated100684689635328 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 100656176 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated61793076 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated100447273039872 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed39792774 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 _runtime2606.493335004 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 100665848 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed105595279816704 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed38872770 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated39793036 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed105971992470016 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated38872772 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 5135424580078 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated105972145943552 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed16754992 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed100447256000512 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5135510383314 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5287456308224ߌ2- _wandbruntime2606  _runtime2606L] 1loss/FBt1/assignment_loss0.14726180332717817 0loss/FBt2/assignment_loss0.5226163669934977 /loss/FRt2/detection_loss2.3771340421267917 0loss/SRqqt1/detection_loss2.104530932574437 1loss/SRqqt2/detection_loss0.9764487322557855 /loss/FBt1/detection_loss0.9665850910653428 /loss/FBt2/detection_loss2.1323015689849854 0loss/FRt1/assignment_loss4.0903766791025795 .loss/FRt1/detection_loss2.423413196064177 &loss/total_loss20.840146860611462 epoch0 % _timestamp1.7857978936717353e+09 _runtime2606.494573443 2loss/SRqqt1/assignment_loss1.3863180731904918 trainer/global_step5849 _step467 0loss/FRt2/assignment_loss2.2270591760438587 2loss/SRqqt2/assignment_loss1.4861011988823376񌐌  2loss/SRqqt1/assignment_loss1.3863180731904918 trainer/global_step5849 _step467 0loss/FRt2/assignment_loss2.2270591760438587 2loss/SRqqt2/assignment_loss1.4861011988823376 1loss/FBt1/assignment_loss0.14726180332717817 0loss/FBt2/assignment_loss0.5226163669934977 /loss/FRt2/detection_loss2.3771340421267917 0loss/SRqqt1/detection_loss2.104530932574437 1loss/SRqqt2/detection_loss0.9764487322557855 /loss/FBt1/detection_loss0.9665850910653428 /loss/FBt2/detection_loss2.1323015689849854 0loss/FRt1/assignment_loss4.0903766791025795 .loss/FRt1/detection_loss2.423413196064177 &loss/total_loss20.840146860611462 epoch0 % _timestamp1.7857978936717353e+09 _runtime2606.494573443$B(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts124.984' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent41.66133333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0؋S(:  V disk.drbd1817.out0 network.sent 5.29342e+06 disk.rbd0.out0 disk.drbd1356.out0 proc.cpu.threads29 disk.sda1.in0 disk.drbd2078.out0 disk.drbd1817.in0 disk.drbd1828.in0 disk.drbd1289.out0 disk.drbd1285.out0 disk.drbd1584.out0$ disk./.usageGB578.3412933349609 disk.drbd2041.out0 disk.drbd1552.out0 network.recv358495 disk.drbd1226.in0 disk.drbd1673.in0 disk.drbd1673.out0 disk.drbd1584.in0 disk.drbd2118.in0 disk.drbd2118.out0 disk.drbd2111.in0 disk.drbd2111.out0! proc.memory.rssMB 11260.40625 disk.drbd2041.in0 disk.drbd1938.out0 disk.drbd1510.in0 disk.drbd1510.out0 disk.drbd1931.in0# disk.nvme0n1.out104.0498046875 disk.drbd1356.in0 disk.drbd1965.out0 disk.drbd1552.in0 disk.sda1.out0 disk.dm-0.in 2.47265625 disk.dm-0.out104.0498046875 disk.sda2.in0 disk.rbd1.out 10.345703125 disk.nvme0n1.in 2.47265625 disk.drbd1965.in0) disk./.usagePercent32.34387271192606 disk.sda2.out 70.671875% memory_percent41.233450174331665* proc.memory.availableMB38513.24609375 cpu13.896349229558574 disk.drbd1931.out0 disk.drbd1938.in0 disk.sda.out 70.671875 disk.rbd1.in0* proc.memory.percent17.182016372680664 disk.drbd1828.out0 disk.rbd0.in0 disk.sda.in0 disk.drbd1285.in0 disk.drbd1226.out0 disk.drbd2078.in0 disk.drbd1550.in0 disk.drbd1550.out0 disk.drbd1289.in0`B(: U gpu.0.gpu41.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts98.389' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent32.79633333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0@Y(:  ɋV disk.drbd1226.in0 disk.drbd1584.out0 disk.drbd2118.in0 disk.drbd1965.in0 disk.drbd1550.in0 disk.drbd2118.out0 disk.drbd1817.in0 disk.drbd1828.out0% memory_percent41.233450174331665 disk.drbd2078.in0 disk.drbd2041.in0 disk.drbd1510.out0 disk.drbd1931.in0 disk.drbd2041.out0 disk.sda.in0 disk.drbd1552.in0$ disk.nvme0n1.out104.21435546875 disk.drbd1226.out0 disk.drbd2111.out0 disk.sda1.out0* proc.memory.percent17.182016372680664 disk.sda2.out 71.1484375 disk.drbd2111.in0 disk.drbd1289.in0 disk.drbd1584.in0 disk.drbd1356.out0 disk.rbd1.out 10.3779296875 disk.rbd0.out0 disk.drbd1285.in0* proc.memory.availableMB38513.24609375 cpu13.894418619532283 proc.cpu.threads29 network.recv360328 disk.drbd2078.out0 disk.rbd0.in0 disk.drbd1285.out0$ disk./.usageGB578.3412933349609 disk.drbd1931.out0 disk.drbd1828.in0 disk.drbd1289.out0 disk.sda1.in0 network.sent 5.341014e+06 disk.drbd1356.in0 disk.drbd1965.out0 disk.drbd1673.out0! disk.dm-0.out104.21435546875! proc.memory.rssMB 11260.40625 disk.sda2.in0 disk.drbd1938.out0 disk.sda.out 71.1484375 disk.dm-0.in 2.47265625 disk.nvme0n1.in 2.47265625 disk.drbd1817.out0 disk.drbd1673.in0 disk.drbd1552.out0 disk.drbd1550.out0 disk.drbd1938.in0) disk./.usagePercent32.34387271192606 disk.rbd1.in0 disk.drbd1510.in0_2- _wandbruntime2628  _runtime2628Sx _step468 ' lr-AdamW/pg10.00015203012807682068 ' lr-AdamW/pg20.00015203012807682068 trainer/global_step5899 $ _timestamp1.785797915207225e+09 _runtime2628.030230436 ' lr-AdamW/pg20.00015203012807682068 trainer/global_step5899 $ _timestamp1.785797915207225e+09 _runtime2628.030230436 _step468 ' lr-AdamW/pg10.00015203012807682068rX2- _wandbruntime2628  _runtime2628s-#'O WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated39198279 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated103950099778320 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed39198277 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current242 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 101499162 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed40125080 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated106479672188928 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 5331760294400 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5331629620224 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 trainer/global_step5899 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed62300885 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated106859598652928 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated16894627 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated101527838358528 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed98771561365472 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated101288444427264 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5191227761664 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 5178521373488 _runtime2628.031444192 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed106479567719936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 5191140332032 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed101527813454848 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 $ _timestamp1.785797915208226e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed39198277 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed101288427387904 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5191140332032 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed103949996935836 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current244 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed23230695 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated106479672188928 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed16894385 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed62300885 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5191227761664 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed106859443075072 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 101508820 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated39198279 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated62310541 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed101288427387904 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated62310541 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated40125324 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5178435570364 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed106479567719936f7&O ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 trainer/global_step5899 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed62300885 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated106859598652928 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated16894627 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated101527838358528 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed98771561365472 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated101288444427264 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5191227761664 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 5178521373488 _runtime2628.031444192 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed106479567719936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 5191140332032 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed101527813454848 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 $ _timestamp1.785797915208226e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed39198277 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed101288427387904 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5191140332032 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed103949996935836 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current244 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed23230695 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated106479672188928 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed16894385 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed62300885 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5191227761664 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed106859443075072 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 101508820 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated39198279 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated62310541 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed101288427387904 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated62310541 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated40125324 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5178435570364 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed106479567719936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated39198279 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated103950099778320 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed39198277 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current242 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 101499162 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed40125080 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated106v8479672188928 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 5331760294400 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5331629620224 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121y2- _wandbruntime2628  _runtime2628Sb&M NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated39204922 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5192107388416 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 5179313043586 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current254 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 5192019951616 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated101305611190272 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed106877551789568 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated98788318577152 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5179398846822 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed62311408 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 101526000 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 101516328 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 % _timestamp1.7857979155758054e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated106877711554560 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated101545045475328 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed39204920 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed101305594150912 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed106497614102528 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated106497718578688 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed98788301537792 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed101545020571648 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed101305594150912 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed39204920 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated40132115 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed103967614581378 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5192019951616 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated39204922 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current256 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated23234634 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5192107388416 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed16897227 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated62321078 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed23234632 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 101526000 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 101516328 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated16897481 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated106497718578688 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5332666079232 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated101305611190272 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed40131859 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 trainer/global_step5899 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _runtime2628.399198442 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated62321078 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 &M D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed39204920 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed101305594150912 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed106497614102528 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated106497718578688 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed98788301537792 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed101545020571648 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed101305594150912 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed39204920 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated40132115 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed103967614581378 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5192019951616 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated39204922 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current256 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated23234634 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5192107388416 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed16897227 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated62321078 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed23234632 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 101526000 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 101516328 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated16897481 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated106497718578688 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5332666079232 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated101305611190272 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed40131859 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 trainer/global_step5899 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _runtime2628.399198442 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated62321078 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated39204922 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5192107388416 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 5179313043586 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current254 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 5192019951616 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated101305611190272 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed106877551789568 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated98788318577152 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5179398846822 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed62311408 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 101526000 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 101516328 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 % _timestamp1.7857979155758054e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated106877711554560 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated101545045475328 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640nG<2- _wandbruntime2628  _runtime2628 1loss/FBt1/assignment_loss0.11180083398391771 /loss/FBt1/detection_loss1.1611480635940183 1loss/FBt2/assignment_loss0.18107539415302654 /loss/FRt1/detection_loss2.2918931915093275 1loss/SRqqt1/detection_loss1.9092563588305242 &loss/total_loss20.343203389176544 % _timestamp1.7857979155777152e+09 _step471 0loss/FRt2/assignment_loss3.1260519990554223 /loss/FRt2/detection_loss2.6526220369224367 /loss/FBt2/detection_loss1.0323533713817596 trainer/global_step5899 0loss/FRt1/assignment_loss3.7803120017051697 1loss/SRqqt2/detection_loss1.2467963520413752 epoch0 _runtime2628.400611199 2loss/SRqqt1/assignment_loss1.3365007010953767 2loss/SRqqt2/assignment_loss1.5133930849041908B}  0loss/FRt2/assignment_loss3.1260519990554223 /loss/FRt2/detection_loss2.6526220369224367 /loss/FBt2/detection_loss1.0323533713817596 trainer/global_step5899 0loss/FRt1/assignment_loss3.7803120017051697 1loss/SRqqt2/detection_loss1.2467963520413752 epoch0 _runtime2628.400611199 2loss/SRqqt1/assignment_loss1.3365007010953767 2loss/SRqqt2/assignment_loss1.5133930849041908 1loss/FBt1/assignment_loss0.11180083398391771 /loss/FBt1/detection_loss1.1611480635940183 1loss/FBt2/assignment_loss0.18107539415302654 /loss/FRt1/detection_loss2.2918931915093275 1loss/SRqqt1/detection_loss1.9092563588305242 &loss/total_loss20.343203389176544 % _timestamp1.7857979155777152e+09 _step471j (: ׵U gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts126.74' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.24666666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors04kZ(:  ȾU cpu13.892556722279181$ disk./.usageGB578.3396873474121 disk.drbd1226.out0 disk.drbd2118.out0 disk.rbd1.in0 disk.drbd1828.in0 disk.drbd1828.out0 disk.drbd1938.in0 disk.drbd2111.out0 disk.drbd1965.out0 network.sent 5.386036e+06 network.recv362240 disk.dm-0.in 2.47265625 disk.drbd1817.out0 proc.cpu.threads29 disk.drbd1931.out0 disk.sda1.in0 disk.sda1.out0 disk.drbd2041.out0 disk.drbd1226.in0 disk.drbd1356.in0 disk.drbd1550.out0! disk.dm-0.out104.46044921875$ disk.nvme0n1.out104.46044921875 disk.sda2.out 71.45703125 disk.drbd1931.in0 disk.drbd1938.out0 disk.drbd1285.out0 disk.drbd2078.out0 disk.rbd1.out 10.4716796875* proc.memory.percent17.182016372680664 disk.drbd1289.in0 disk.drbd2118.in0 disk.drbd2111.in0 disk.drbd1965.in0$ memory_percent41.23404622077942 disk.drbd1673.in0 disk.drbd2041.in0 disk.sda2.in0 disk.drbd1510.out0 disk.drbd2078.in0 disk.drbd1584.in0 disk.drbd1673.out0 disk.drbd1817.in0* proc.memory.availableMB38512.85546875 disk.rbd0.in0 disk.drbd1285.in0 disk.rbd0.out0 disk.sda.in0 disk.sda.out 71.45703125 disk.drbd1552.in0 disk.drbd1584.out0! proc.memory.rssMB 11260.40625) disk./.usagePercent32.34378289669506 disk.nvme0n1.in 2.47265625 disk.drbd1552.out0 disk.drbd1550.in0 disk.drbd1289.out0 disk.drbd1510.in0 disk.drbd1356.out0z$Sf2- _wandbruntime2649  _runtime2649L  % _timestamp1.7857979371234446e+09 _runtime2649.947723536 _step472 % lr-AdamW/pg10.000153318737401086 % lr-AdamW/pg20.000153318737401086 trainer/global_step5949 m % lr-AdamW/pg10.000153318737401086 % lr-AdamW/pg20.000153318737401086 trainer/global_step5949 % _timestamp1.7857979371234446e+09 _runtime2649.947723536 _step4722- _wandbruntime2649  _runtime2649UO _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5376762848256 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed62828887 % _timestamp1.7857979371241255e+09 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed107764932143104 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5235136467456 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated23427547 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed102146765538304 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 _step473 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed39530427 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated62838543 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed107381902005760 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5235223897088 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 trainer/global_step5949 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 102368972 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed40463512 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current275 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated99608587020832 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated107382006474752 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated62838543 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed62828887 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5235223897088 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed104830894015344 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5222324033872 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed39530427 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated17036242 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated104830996857828 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed99608569981472 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated39530429 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed102146765538304 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 5235136467456 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated102388194198528 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated102146782577664 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated107382006474752 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed102388169294848 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current277 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed107381902005760 ^JDeviceStatsMonit~or.on_train_batch_start/inactive_split_bytes.all.allocated107765087720960 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 5222409836996U'O trainer/global_step5949 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 102368972 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed40463512 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current275 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated99608587020832 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated107382006474752 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated62838543 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed62828887 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5235223897088 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed104830894015344 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5222324033872 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed39530427 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated17036242 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated104830996857828 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed99608569981472 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated39530429 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed102146765538304 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 5235136467456 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated102388194198528 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated102146782577664 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated107382006474752 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed102388169294848 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current277 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed107381902005760 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated107765087720960 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 5222409836996 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5376762848256 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed62828887 % _timestamp1.7857979371241255e+09 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed107764932143104 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5235136467456 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated23427547 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed102146765538304 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 _step473 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed39530427 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated62838543 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed107381902005760 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5235223897088 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484-2- _wandbruntime2650  _runtime2650KQ-^&M A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated107400052864512 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed102163932301312 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 124375552 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated40470541 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated23431484 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 102386152 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated62849080 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5236016087040 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5223287310330 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed102163932301312 _step474 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed107783047419392 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed104848511660886 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed99625310153792 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated104848614503482 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current256 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 % _timestamp1.7857979374954035e+09 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current254 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated17039057 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated39537072 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed62839410 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 _runtime2650.318804723 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated39537072 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed62839410 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed39537070 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated99625327193152 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated107400052864512 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated102163949340672 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated102163949340672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed40470285 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0em&M SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed102163932301312 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 124375552 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated40470541 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated23431484 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 102386152 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated62849080 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5236016087040 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5223287310330 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed102163932301312 _step474 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed107783047419392 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed104848511660886 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed99625310153792 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated104848614503482 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current256 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 % _timestamp1.7857979374954035e+09 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current254 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated17039057 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated39537072 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed62839410 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 _runtime2650.318804723 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated39537072 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed62839410 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed39537070 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated99625327193152 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated107400052864512 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated102163949340672 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated102163949340672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed40470285 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated107400052864512 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136ٌ2- _wandbruntime2650  _runtime2650 _step475 0loss/FRt2/assignment_loss2.6747188475625268 0loss/FBt2/assignment_loss0.3871067427098751 .loss/FRt2/detection_loss2.384322822736255 0loss/FRt1/assignment_loss3.6245958425781946 1loss/FBt1/assignment_loss0.10608162448823863 -loss/FRt1/detection_loss2.64642936702479 0loss/SRqqt1/detection_loss2.346221211958133 /loss/FBt1/detection_loss1.2654427478728827 /loss/FBt2/detection_loss0.8740220963954926 %loss/t4fe&otal_loss20.57072125577887 trainer/global_step5949 2loss/SRqqt2/assignment_loss1.4914746935364236 1loss/SRqqt2/detection_loss0.9236934325458963 $ _timestamp1.785797937496654e+09 _runtime2650.319536477 epoch0 2loss/SRqqt1/assignment_loss1.8466118263701599vs 1loss/FBt1/assignment_loss0.10608162448823863 -loss/FRt1/detection_loss2.64642936702479 0loss/SRqqt1/detection_loss2.346221211958133 /loss/FBt1/detection_loss1.2654427478728827 /loss/FBt2/detection_loss0.8740220963954926 %loss/total_loss20.57072125577887 trainer/global_step5949 2loss/SRqqt2/assignment_loss1.4914746935364236 1loss/SRqqt2/detection_loss0.9236934325458963 $ _timestamp1.785797937496654e+09 _runtime2650.319536477 epoch0 2loss/SRqqt1/assignment_loss1.8466118263701599 _step475 0loss/FRt2/assignment_loss2.6747188475625268 0loss/FBt2/assignment_loss0.3871067427098751 .loss/FRt2/detection_loss2.384322822736255 0loss/FRt1/assignment_loss3.6245958425781946 (: òU gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts119.06' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent39.68666666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0xUT(:  U disk.drbd1285.in0 disk.drbd1226.out0 disk.drbd2118.out0 disk.sda2.in0 network.recv366412 disk.sda1.in0# disk.nvme0n1.out105.3759765625 disk.drbd2111.in0 disk.drbd1817.in0 network.sent 5.47615e+06 disk.drbd1828.in0 disk.drbd1938.out0 disk.drbd1226.in0 disk.drbd2111.out0 disk.drbd1965.out0' proc.memory.availableMB 38512.78125 disk.drbd1550.in0 disk.sda.in0 disk.drbd1938.in0 disk.sda1.out0 disk.dm-0.in 2.47265625! proc.memory.rssMB 11260.40625 disk.drbd1510.in0 disk.drbd2041.in0 disk.drbd2041.out0 cpu13.890669411888682 disk.drbd1584.out0 disk.drbd1673.out0 disk.drbd1931.in0 disk.drbd1828.out0 disk.sda.out 71.72265625 disk.drbd1285.out0 disk.drbd1673.in0 disk.rbd0.in0 disk.rbd1.in0$ memory_percent41.23415946960449$ disk./.usageGB578.3417053222656 disk.drbd1552.out0 disk.rbd0.out0 disk.drbd1356.out0 disk.drbd1817.out0 disk.rbd1.out 10.5341796875 disk.drbd1289.in0 disk.drbd2078.out0 disk.nvme0n1.in 2.47265625 disk.drbd1931.out0 disk.drbd1552.in0 disk.drbd1550.out0 disk.drbd2078.in0 disk.drbd1584.in0 disk.drbd1965.in0) disk./.usagePercent32.34389575241286 disk.drbd2118.in0 disk.drbd1289.out0 disk.drbd1510.out0 disk.drbd1356.in0 disk.sda2.out 71.72265625* proc.memory.percent17.182016372680664 disk.dm-0.out105.3759765625 proc.cpu.threads29~(: źU gpu.0.gpu41.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts118.927' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent39.64233333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0LqW(:  źU disk.drbd1931.in0 disk.drbd1931.out0 disk.drbd1828.out0 disk.drbd1817.in0 disk.drbd1510.out0 disk.drbd1550.out0) disk./.usagePercent32.34389681910208 disk.drbd1552.in0 disk.drbd1828.in0* proc.memory.percent17.182016372680664 disk.rbd1.in0 disk.drbd1817.out0 network.recv368127 disk.drbd1550.in0 disk.rbd0.out0 disk.drbd2078.in0 disk.drbd2078.out0 disk.drbd1584.in0$ disk.nvme0n1.out106.06982421875# memory_percent41.2345290184021# disk./.usageGB578.341724395752 disk.sda.in0 disk.drbd1356.in0 disk.drbd2111.in0 disk.sda1.in0 disk.drbd2041.in0 disk.drbd1552.out0 disk.drbd2118.in0 cpu13.888986765869982 disk.drbd1289.out0 disk.drbd1226.out0 disk.sda2.in0! proc.memory.rssMB 11260.40625! disk.dm-0.out106.06982421875 disk.drbd1673.in0 proc.cpu.threads29 disk.drbd1938.out0 disk.drbd1285.in0 disk.drbd1965.out0 disk.drbd2111.out0 disk.drbd1673.out0 disk.drbd1289.in0 disk.drbd1226.in0 disk.drbd2041.out0 disk.dm-0.in 2.47265625 disk.drbd1584.out0 disk.sda2.out 72.27734375 disk.drbd1965.in0 disk.sda1.out0 network.sent 5.477336e+06 disk.rbd0.in0 disk.nvme0n1.in 2.47265625 disk.drbd1938.in0 disk.sda.out 72.27734375 disk.drbd2118.out0) proc.memory.availableMB 38512.5390625 disk.drbd1285.out0 disk.drbd1510.in0 disk.drbd1356.out0 disk.rbd1.out 10.5341796875w=2- _wandbruntime2671  _runtime2671?J trainer/global_step5999 % _timestamp1.7857979589852583e+09 _runtime2671.808225421 _step476 ' lr-AdamW/pg10.00015460734672535128 ' lr-AdamW/pg20.00015460734672535128_PI ' lr-AdamW/pg20.00015460734672535128 trainer/global_step5999 % _timestamp1.7857979589852583e+09 _runtime2671.808225421 _step476 ' lr-AdamW/pg10.00015460734672535128;t2- _wandbruntime2671  _runtime26716^'O JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5421966455808 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed23624395 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated108670649265664 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated103005120728064 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed100445578597472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed40803928 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated103248550038528 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 _step477 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed108670491590656 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed105711791094852 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 % _timestamp1.7857979589861798e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed39862577 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed17179533 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5266212497380 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 _runtime2671.812645626 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 5279132602880 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated63366545 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated100445595636832 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 5266298300504 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated23624397 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated108284340760576 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 103219466 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated39862579 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated63366545 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated17179778 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed39862577 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed63356889 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed108284236291584 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated40804175 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed63356889 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current245 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 trainer/global_step5999 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 103229124 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 5422099227136 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated103005120728064 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed103005103688704 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed103005103688704 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5279132602880 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed108284236291584 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated108284340760576 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5279220032512 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current247 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated105711893937336'O \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed103005103688704 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed103005103688704 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5279132602880 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed108284236291584 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated108284340760576 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5279220032512 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current247 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated105711893937336 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5421966455808 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed23624395 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated108670649265664 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated103005120728064 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed100445578597472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed40803928 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated103248550038528 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 _step477 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed108670491590656 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed105711791094852 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 % _timestamp1.7857979589861798e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed39862577 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed17179533 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5266212497380 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 _runtime2671.812645626 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 5279132602880 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated63366545 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated100445595636832 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 5266298300504 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated23624397 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated108284340760576 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 103219466 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated39862579 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated63366545 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated17179778 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed39862577 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed63356889 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed108284236291584 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated40804175 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed63356889 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current245 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 trainer/global_step5999 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 103229124 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 5422099227136 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated103005120728064?M2- _wandbruntime2672  _runtime267210M [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated103022287491072 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed39869220 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5280099659264 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5280099659264 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 103246304 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 5267089970602 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed63367412 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed39869220 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 trainer/global_step5999 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5267175773838 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed108302282674176 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 5422870883840 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed103022270451712 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 103236632 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed105729408740394 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5280012222464 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated39869222 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated63377082 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed23628332 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated40810915 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current271 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated105729511582990 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated63377082 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated108688760803328 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed40810644 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 J;DeviceStatsMonitor.on_train_batch_end/active_byte4 s.all.peak 2940455936 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 5280012222464 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 103246304 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated39869222 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed103022270451712 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated100462335809152 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 _runtime 2672.1737678 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated103022287491072 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated23628334 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5423003648000 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _step478 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated17182581 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current269 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed103265732251648 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed63367412 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 103236632 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed17182312 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed108688603135488 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated108302387150336 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed40810644 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 5280012222464 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 103246304 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated39869222 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed103022270451712 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated100462335809152 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 _runtime 2672.1737678 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated103022287491072 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated23628334 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5423003648000 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _step478 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated17182581 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current269 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed103265732251648 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed63367412 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 103236632 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed17182312 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed108688603135488 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated108302387150336 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5280099659264 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5280099659264 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 103246304 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 5267089970602 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed63367412 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed39869220 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 trainer/global_step5999 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5267175773838 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed108302282674176 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 5422870883840 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed103022270451712 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 103236632 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed105729408740394 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5280012222464 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated39869222 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated63377082 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed23628332 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated40810915q2- _wandbruntime2672  _runtime2672;L/n .loss/FBt2/detection_loss2.607370142279656 0loss/FRt1/assignment_loss3.8574561790986492 0loss/FRt2/assignment_loss3.0197857054797086 2loss/SRqqt2/assignment_loss1.3593037775212742 1loss/FBt1/assignment_loss0.21852576079966446 0loss/FBt2/assignment_loss0.2463042386885458 /loss/FRt1/detection_loss2.3166479191996836 1loss/SRqqt2/detection_loss0.9412240162042969 .loss/FRt2/detection_loss2.700605557113886 %loss/total_loss21.84606403154643 /loss/FBt1/detection_loss1.0679513130125997 trainer/global_step5999 _step479 epoch0 % _timestamp1.7857979593521442e+09 2loss/SRqqt1/assignment_loss1.3993739667865965 /loss/SRqqt1/detection_loss2.11151545536187 _runtime2672.174985772@o %loss/total_loss21.84606403154643 /loss/FBt1/detection_loss1.0679513130125997 trainer/global_step5999 _step479 epoch0 % _timestamp1.7857979593521442e+09 2loss/SRqqt1/assignment_loss1.3993739667865965 /loss/SRqqt1/detection_loss2.11151545536187 _runtime2672.174985772 .loss/FBt2/detection_loss2.607370142279656 0loss/FRt1/assignment_loss3.8574561790986492 0loss/FRt2/assignment_loss3.0197857054797086 2loss/SRqqt2/assignment_loss1.3593037775212742 1loss/FBt1/assignment_loss0.21852576079966446 0loss/FBt2/assignment_loss0.2463042386885458 /loss/FRt1/detection_loss2.3166479191996836 1loss/SRqqt2/detection_loss0.9412240162042969 .loss/FRt2/detection_loss2.700605557113886E W(:  Ժ݆U network.recv368325 disk.drbd1817.out0 disk.drbd1828.out0 disk.drbd1285.in0) proc.memory.availableMB 38513.2578125 disk.drbd1356.in0 disk.sda2.out 72.72265625 disk.drbd1931.in0 disk.drbd2111.out0 disk.drbd1552.out0 disk.drbd1226.out0 disk.nvme0n1.in 2.47265625 disk.sda.in0 disk.drbd1510.in0 disk.drbd2078.in0 disk.drbd2078.out0 disk.drbd1673.in0 disk.drbd1550.out0 disk.drbd1289.out0! disk.dm-0.out107.37353515625 disk.drbd1356.out0 disk.sda2.in0 disk.drbd1965.out0 disk.drbd1550.in0 disk.dm-0.in 2.47265625 disk.drbd1673.out0 disk.drbd2118.in0 disk.drbd1965.in0 disk.drbd1552.in0 cpu13.887273203251572 disk.rbd1.in0 disk.drbd1931.out0 disk.drbd1510.out0 proc.cpu.threads29 network.sent 5.480184e+06 disk.drbd1584.out0 disk.drbd2118.out0$ disk.nvme0n1.out107.37353515625 disk.drbd2111.in0 disk.rbd0.out0 disk.sda1.out0 disk.rbd1.out 10.56640625 disk.drbd2041.in0 disk.drbd1828.in0 disk.drbd1938.in0 disk.rbd0.in0 disk.sda1.in0! proc.memory.rssMB 11260.40625* proc.memory.percent17.182016372680664$ disk./.usageGB578.3417434692383 disk.drbd1584.in0 disk.drbd1817.in0 disk.drbd1289.in0 disk.sda.out 72.72265625 disk.drbd2041.out0 disk.drbd1938.out0 disk.drbd1285.out0 disk.drbd1226.in0$ memory_percent41.23343229293823) disk./.usagePercent32.343897885791288T՛(: Ժ㣨V gpu.0.gpu24.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts127.727' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.57566666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0†`2- _wandbruntime2693  _runtime2693gc _step480 & lr-AdamW/pg10.0001558959560496166 & lr-AdamW/pg20.0001558959560496166 trainer/global_step6049 $ _timestamp1.785797981026456e+09 _runtime2693.849456535_ & lr-AdamW/pg10.0001558959560496166 & lr-AdamW/pg20.0001558959560496166 trainer/global_step6049 $ _timestamp1.785797981026456e+09 _runtime2693.849456535 _step480T{l2- _wandbruntime2693  _runtime2693['O PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed40194727 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 104079618 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated109186675046400 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed109576066285568 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed101282587213472 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 5467311790592 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 # _timestamp1.78579798102745e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5323216167936 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed109186570577408 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed17323930 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 5310186764012 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 5323128738304 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current250 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed40194727 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated103863458878464 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated109576217669120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated40194729 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed109186570577408 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 126479872 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 104089276 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed103863441839104 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated41145427 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 _step481 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5323128738304 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated40194729 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed23821245 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed63884891 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated109186675046400 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated63894547 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated106592791016844 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated17324180 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed106592688174360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated63894547 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 _runtime2693.850739782 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5467185310720 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed109186570577408 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed17323930 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 5310186764012 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 5323128738304 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current250 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed40194727 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated103863458878464 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated109576217669120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated40194729 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed109186570577408 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 126479872 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 UFDeviceStatsMon]itor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 104089276 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed103863441839104 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated41145427 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 _step481 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5323128738304 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated40194729 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed23821245 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed63884891 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated109186675046400 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated63894547 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated106592791016844 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated17324180 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed106592688174360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated63894547 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 _runtime2693.850739782 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5467185310720 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 5467311790592 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 # _timestamp1.78579798102745e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5323216167936@1 2- _wandbruntime2694  _runtime2694/T&M REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed17326822 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated101299344425152 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated103880625641472 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed63895414 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5311064237346 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated109204721436160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed40201370 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5324095794688 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated106610408662498 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 104106456 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated40201372 % _timestamp1.7857979813963916e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 104096784 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 5324008357888 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5468219435008 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 trainer/global_step6049 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 104106456 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated63905084 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed63895414 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated109204721436160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated104126112995328 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed41152004 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed109204616960000 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated41152275 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed106610305819902 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed101299327385792 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 _runtime2694.219807511 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5324008357888 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current271 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated40201372 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed40201370 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated63905084 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 104106456 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated63905084 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed63895414 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated109204721436160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated104126112995328 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed41152004 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed109204616960000 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated41152275 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed106610305819902 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed101299327385792 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 _runtime2694.219807511 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5324008357888 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current271 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated40201372 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed40201370 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated63905084 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed63895414 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5311064237346 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated109204721436160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed40201370 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5324095794688 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated106610408662498 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 104106456R}w2- _wandbruntime2694  _runtime2694By %loss/total_loss20.36733482022587 /loss/FRt1/assignment_loss4.005033612251282 2loss/SRqqt1/assignment_loss1.4193329497387535 1loss/SRqqt2/assignment_loss1.478742322758024 0loss/FBt2/assignment_loss0.4656804483383894 /loss/FRt1/detection_loss2.4184157419772374 % _timestamp1.7857979813976326e+09 _runtime 2694.22044777 0loss/FBt1/assignment_loss0.0736971140171559 1loss/SRqqt1/detection_loss2.3363850185353505 1loss/SRqqt2/detection_loss0.9820127354497895 /loss/FBt2/detection_loss1.1782978296279907 0loss/FRt2/assignment_loss2.5086162717718827 .loss/FRt2/detection_loss2.505306509241723 .loss/FBt1/detection_loss0.995814266518289 trainer/global_step6049 _step483 epoch0O %loss/total_loss20.36733482022587 /loss/FRt1/assignment_loss4.005033612251282 2loss/SRqqt1/assignment_loss1.4193329497387535 1loss/SRqqt2/assignment_loss1.478742322758024 0loss/FBt2/assignment_loss0.4656804483383894 /loss/FRt1/detection_loss2.4184157419772374 % _timestamp1.7857979813976326e+09 _runtime 2694.22044777 0loss/FBt1/assignment_loss0.0736971140171559 1loss/SRqqt1/detection_loss2.3363850185353505 1loss/SRqqt2/detection_loss0.9820127354497895 /loss/FBt2/detection_loss1.1782978296279907 0loss/FRt2/assignment_loss2.5086162717718827 .loss/FRt2/detection_loss2.505306509241723 .loss/FBt1/detection_loss0.995814266518289 trainer/global_step6049 _step483 epoch0#S(:  U) disk./.usagePercent32.34395250027853 disk.dm-0.in 2.47265625* proc.memory.percent17.182016372680664 cpu13.88549011547609$ disk./.usageGB578.3427200317383 disk.drbd2041.in0 disk.drbd1289.in0 disk.sda.in0 disk.drbd1510.in0 disk.drbd2078.in0 disk.drbd1584.out0 disk.rbd1.in0 network.sent 5.52767e+06 disk.sda.out 73.2890625 disk.drbd2111.in0 disk.drbd1817.out0! proc.memory.rssMB 11260.40625 disk.drbd1289.out0 disk.drbd1817.in0 network.recv371699 disk.drbd2041.out0 disk.drbd1226.out0 disk.sda2.out 73.2890625 disk.rbd0.in0 disk.drbd1673.out0 disk.rbd0.out0 disk.drbd1356.out0$ memory_percent41.23347997665405 disk.drbd1931.in0 disk.drbd1552.in0 disk.sda1.in0 disk.drbd1510.out0 disk.drbd2118.out0 disk.drbd1965.out0 disk.drbd1673.in0 disk.drbd1938.in0 disk.drbd1584.in0 disk.drbd1965.in0 disk.rbd1.out 10.66015625 disk.drbd1828.out0 disk.drbd1285.in0 proc.cpu.threads29 disk.drbd1550.out0 disk.drbd1226.in0 disk.drbd1356.in0 disk.drbd1828.in0 disk.drbd1938.out0 disk.drbd1285.out0 disk.nvme0n1.in 2.47265625 disk.sda2.in0 disk.drbd1552.out0! disk.dm-0.out108.55126953125$ disk.nvme0n1.out108.55126953125 disk.drbd1931.out0 disk.drbd2078.out0 disk.drbd2118.in0 disk.sda1.out0 disk.drbd1550.in0 disk.drbd2111.out0) proc.memory.availableMB 38513.2265625 (: V gpu.0.gpu41.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts132.366' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent44.12200000000001 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0q̶(: U gpu.0.gpu41.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts127.534' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.51133333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0wCS(:  U disk.drbd1550.in0 disk.drbd2118.out0 disk.sda2.in0 disk.drbd1965.in0 disk.drbd1965.out0 disk.drbd2041.out0 disk.drbd1828.in0 network.recv374285 disk.drbd1817.in0 disk.rbd1.out 10.89453125$ disk./.usageGB578.3432693481445 disk.sda.in0 disk.drbd1285.in0$ memory_percent41.23462438583374# disk.nvme0n1.out110.2958984375 disk.drbd1673.in0 disk.drbd1550.out0 disk.drbd1285.out0 disk.drbd1938.in0 disk.sda1.in0 disk.drbd2078.out0 disk.drbd1584.in0 disk.drbd1356.out0 disk.drbd1289.in0 disk.rbd0.in0 disk.drbd1510.out0 disk.drbd1931.out0 disk.dm-0.out110.2958984375 disk.drbd1938.out0 disk.drbd1226.in0 disk.drbd2078.in0) proc.memory.availableMB 38512.4765625* proc.memory.percent17.182016372680664 disk.sda2.out 73.6015625 disk.drbd2111.in0 disk.drbd2111.out0 disk.drbd2041.in0 disk.drbd1289.out0 disk.sda1.out0 disk.drbd1510.in0 disk.drbd1226.out0 disk.drbd1552.in0 disk.drbd2118.in0 disk.drbd1817.out0 cpu13.883950967741868 disk.drbd1356.in0 disk.drbd1673.out0 disk.drbd1931.in0 disk.rbd0.out0) disk./.usagePercent32.34398322092761 disk.sda.out 73.6015625 disk.drbd1584.out0 disk.drbd1552.out0 disk.dm-0.in 2.47265625 network.sent 5.572449e+06 disk.drbd1828.out0! proc.memory.rssMB 11260.40625 disk.nvme0n1.in 2.47265625 proc.cpu.threads29 disk.rbd1.in0`2- _wandbruntime2715  _runtime2715u & lr-AdamW/pg10.0001571845653738819 & lr-AdamW/pg20.0001571845653738819 trainer/global_step6099 $ _timestamp1.785798002457669e+09 _runtime2715.280627536 _step484={ & lr-AdamW/pg20.0001571845653738819 trainer/global_step6099 $ _timestamp1.785798002457669e+09 _runtime2715.280627536 _step484 & lr-AdamW/pg10.0001571845653738819=Eb2- _wandbruntime2715  _runtime2715zO F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 5367124873728 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated102119612868832 _step485 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5367212303360 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed107473585253868 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5367124873728 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed24018095 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed17467649 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated64422549 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated104969261718528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 104939770 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5367212303360 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 % _timestamp1.7857980024585915e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed110088904863232 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed64412893 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated17467897 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5512362265600 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed104721779989504 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed110088904863232 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current248 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 5512488745472 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated40526879 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed104721779989504 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current250 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed110481599080448 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5367212303360 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 % _timestamp1.7857980024585915e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed110088904863232 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed64412893 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated17467897 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5512362265600 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed104721779989504 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed110088904863232 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current248 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 5512488745472 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated40526879 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed104721779989504 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current250 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed110481599080448 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed107473585253868 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5367124873728 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed24018095 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed17467649d/x2- _wandbruntime2715  _runtime2715qƚ&M J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed41492555 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated104738963791872 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated102136353041152 _step486 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated110107055721984 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 104966608 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5368004493312 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 104966608 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed104738946752512 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5368091930112 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated64433086 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated104738963791872 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed110106951245824 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 trainer/global_step6099 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 5513268311552 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated64433086 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed64423416 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 % _timestamp1.7857980028281786e+09 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed102136336001792 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed110499712243200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed104738946752512 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed104986443931648 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5368091930112 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated104986468835328 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5513403172864 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _runtime2715.651585309 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887;EM J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed104738946752512 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5368091930112 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated64433086 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated104738963791872 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed110106951245824 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 trainer/global_step6099 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 5513268311552 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated64433086 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed64423416 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 % _timestamp1.7857980028281786e+09 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed102136336001792 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed110499712243200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed104738946752512 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed104986443931648 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 YGDeviceStatsMonitor.on_tsArain_batch_end/active_bytes.small_pool.allocated 5368091930112 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated104986468835328 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5513403172864 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _runtime2715.651585309 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed41492555 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated104738963791872 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated102136353041152 _step486 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated110107055721984 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 104966608 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5368004493312 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 104966608 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0c2- _wandbruntime2715  _runtime2715 1loss/FBt1/assignment_loss0.22474168004667014 /loss/FRt1/detection_loss2.4871985161304475 .loss/FBt1/detection_loss1.752111824398691 -loss/FBt2/detection_loss0.06707763671875 &loss/total_loss20.076508267724414 0loss/FRt1/assignment_loss3.7796109986305235 0loss/FRt2/assignment_loss2.6962967725361096 2loss/SRqqt1/assignment_loss1.6977961457544757 2loss/SRqqt2/assignment_loss1.3710838640005225 .loss/FRt2/detection_loss2.049787110484698 /loss/FBt2/assignment_loss0.675212215970863 0loss/SRqqt1/detection_loss2.290197879076004 _step487 1loss/SRqqt2/detection_loss0.9853936239766578 epoch0 trainer/global_step6099 $ _timestamp1.785798002829854e+09 _runtime2715.652687083,  0loss/FRt2/assignment_loss2.6962967725361096 2loss/SRqqt1/assignment_loss1.6977961457544757 2loss/SRqqt2/assignment_loss1.3710838640005225 .loss/FRt2/detection_loss2.049787110484698 /loss/FBt2/assignment_loss0.675212215970863 0loss/SRqqt1/detection_loss2.290197879076004 _step487 1loss/SRqqt2/detection_loss0.9853936239766578 epoch0 trainer/global_step6099 $ _timestamp1.785798002829854e+09 _runtime2715.652687083 1loss/FBt1/assignment_loss0.22474168004667014 /loss/FRt1/detection_loss2.4871985161304475 .loss/FBt1/detection_loss1.752111824398691 -loss/FBt2/detection_loss0.06707763671875 &loss/total_loss20.076508267724414 0loss/FRt1/assignment_loss3.7796109986305235H(: řU gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts129.622' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent43.20733333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0w-7T(:  U disk.drbd2111.in0 disk.rbd1.in0 disk.rbd1.out 10.89453125 disk.sda1.out0 disk.drbd1510.out0 network.recv375261 disk.drbd1552.out0 disk.drbd1828.in0 disk.drbd1289.out0 disk.drbd1226.in0) disk./.usagePercent32.34398642099523 disk.drbd2118.in0 disk.drbd1550.out0 disk.drbd2078.out0! proc.memory.rssMB 11260.40625 disk.drbd1584.out0 disk.drbd1817.in0 disk.drbd2041.in0 disk.drbd1938.out0 disk.drbd1285.in0 disk.dm-0.in 2.47265625 disk.dm-0.out111.3544921875* proc.memory.percent17.182016372680664 disk.drbd2078.in0$ memory_percent41.23471975326538$ disk./.usageGB578.3433265686035 disk.sda2.in0 disk.drbd1552.in0 disk.nvme0n1.in 2.47265625 disk.drbd1356.out0 disk.sda2.out 74.11328125 disk.drbd1817.out0 disk.rbd0.in0 disk.drbd1510.in0 disk.drbd1356.in0 disk.sda.out 74.11328125 cpu13.882294763111688 disk.drbd1584.in0# disk.nvme0n1.out111.3544921875 disk.drbd1673.in0 disk.drbd1673.out0 disk.drbd1931.out0 network.sent 5.57401e+06 disk.drbd2118.out0 disk.drbd1965.in0 disk.drbd1285.out0 disk.drbd1828.out0 disk.drbd1550.in0 disk.drbd1289.in0 disk.rbd0.out0 proc.cpu.threads29 disk.drbd1931.in0 disk.drbd2041.out0 disk.drbd1938.in0 disk.sda1.in0 disk.drbd1226.out0 disk.drbd1965.out0 disk.drbd2111.out0) proc.memory.availableMB 38512.4140625 disk.sda.in0%+2- _wandbruntime2737  _runtime2737V & lr-AdamW/pg10.0001584731746981472 & lr-AdamW/pg20.0001584731746981472 trainer/global_step6149 % _timestamp1.7857980245303931e+09 _runtime 2737.35337608 _step488S  & lr-AdamW/pg10.0001584731746981472 & lr-AdamW/pg20.0001584731746981472 trainer/global_step6149 % _timestamp1.7857980245303931e+09 _runtime 2737.35337608 _step488Vk(2- _wandbruntime2737  _runtime2737L''O G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated40859029 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated105829617558528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated102956621484832 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed105580118139904 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated105580135179264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed40859027 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated40859029 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed105829592654848 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5557546551808 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed108354482333376 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed110991239149056 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed17611450 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5411121009152 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121UR'O \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed102956604445472 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current251 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 _step489 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 5411121009152 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed64940895 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated110991343618048 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 _runtime2737.357649895 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 5557683517440 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated108354585175860 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated64950551 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 5397963691028 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed64940895 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5411208438784 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated41826648 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 trainer/global_step6149 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed40859027 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated110991343618048 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5411208438784 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed41826395 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5397877887904 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed108354482333376 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed110991239149056 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed17611450 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5411121009152 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated40859029 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated105829617558528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated102956621484832 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed105580118139904 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated105580135179264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed40859027 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated40859029 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed105829592654848 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5557546551808 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed64951418 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated108372202821514 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed108372099978918 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated111405410299392 E?DeviceStmuatsMonitor.on_train_batch_end/oversize_allocations.peak0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _runtime2737.722521931 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated24218884 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed24218882 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 105817088 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 105826760 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated105597301942272 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 5412000628736 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 5558446568448 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5558585624064 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed64951418 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed105846799771648 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated111009390007808 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5412000628736 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5412088065536 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 105826760 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated40865672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5412088065536 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 5398755361126 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated105597301942272 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 % _timestamp1.7857980248991733e+09 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 163959296 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed111009285531648 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed41833196 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated17614566 _step490 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed40865670 trainer/global_step6149&&M NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed64951418 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated108372202821514 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed108372099978918 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated111405410299392 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _runtime2737.722521931 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated24218884 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed24218882 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 105817088 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 105826760 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated105597301942272 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 5412000628736 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 5558446568448 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5558585624064 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed64951418 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed105846799771648 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated111009390007808 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5412000628736 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5412088065536 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 105826760 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated40865672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5412088065536 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 5398755361126 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated105597301942272 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 % _timestamp1.7857980248991733e+09 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 163959296 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed111009285531648 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed41833196 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated17614566 _step490 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed40865670 trainer/global_step6149 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated64961088 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated111009390007808 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed105597284902912 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current254 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed111405246340096 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed40865670 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0Y2- _wandbruntime2737  _runtime2737n7 /loss/FRt1/assignment_loss3.659667783570521 1loss/FBt2/assignment_loss0.29976304653748936 0loss/SRqqt1/detection_loss2.178387605452112 1loss/SRqqt2/detection_loss1.1104324786296857 /loss/FBt1/detection_loss1.1965965704238486 /loss/FRt2/assignment_loss2.917047900147736 2loss/SRqqt1/assignment_loss1.3309915640524455 1loss/SRqqt2/assignment_loss1.260935534887454 .loss/FRt1/detection_loss2.335296355983586 .loss/FRt2/detection_loss2.444113546749577 /loss/FBt2/detection_loss1.0391870207256741 &loss/total_loss19.970846628132808 % _timestamp1.7857980249008005e+09 _runtime2737.723698538 _step491 1loss/FBt1/assignment_loss0.19842722097267879 epoch0 trainer/global_step6149 /loss/FRt1/assignment_loss3.659667783570521 1loss/FBt2/assignment_loss0.29976304653748936 0loss/SRqqt1/detection_loss2.178387605452112 1loss/SRqqt2/detection_loss1.1104324786296857 /loss/FBt1/detection_loss1.1965965704238486 /loss/FRt2/assignment_loss2.917047900147736 2loss/SRqqt1/assignment_loss1.3309915640524455 1loss/SRqqt2/assignment_loss1.260935534887454 .loss/FRt1/detection_loss2.335296355983586 .loss/FRt2/detection_loss2.444113546749577 /loss/FBt2/detection_loss1.0391870207256741 &loss/total_loss19.970846628132808 % _timestamp1.7857980249008005e+09 _runtime2737.723698538 _step491 1loss/FBt1/assignment_loss0.19842722097267879 epoch0 trainer/global_step6149(: U gpu.0.gpu41.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts134.261' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent44.75366666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0mV(:  U disk.drbd1931.out0 disk.rbd0.out0 disk.sda1.in0 disk.dm-0.in 2.47265625 disk.drbd2118.in0 disk.drbd1817.in0 disk.drbd1289.out0 disk.drbd1584.out0! proc.memory.rssMB 11260.40625 disk.drbd1356.in0 disk.drbd1356.out0* proc.memory.percent17.182016372680664) disk./.usagePercent32.34398748768443 disk.drbd1510.out0 disk.drbd1817.out0% memory_percent41.235148906707764) proc.memory.availableMB 38512.1328125 disk.rbd0.in0 disk.sda2.out 74.58203125 proc.cpu.threads29 disk.drbd1938.in0 disk.sda.in0 disk.drbd1226.in0 disk.drbd1226.out0 network.recv377839 disk.drbd1510.in0$ disk./.usageGB578.3433456420898 disk.drbd1965.in0 disk.drbd1965.out0 disk.drbd2078.out0 disk.drbd2118.out0 disk.drbd2078.in0 disk.drbd1673.out0 disk.drbd1931.in0 disk.dm-0.out111.5029296875 disk.sda2.in0 disk.drbd2041.out0 disk.sda1.out0 disk.drbd1552.in0 disk.drbd1550.out0 disk.sda.out 74.58203125 disk.rbd1.in0 disk.drbd1673.in0 disk.drbd2041.in0 disk.drbd1828.in0 disk.drbd1289.in0 disk.nvme0n1.in 2.47265625 disk.drbd2111.in0 network.sent 5.619075e+06 disk.drbd1550.in0 disk.drbd1285.in0 disk.drbd1285.out0 disk.drbd2111.out0 cpu13.880612017679624 disk.drbd1552.out0# disk.nvme0n1.out111.5029296875 disk.drbd1828.out0 disk.drbd1938.out0 disk.drbd1584.in0 disk.rbd1.out 10.9570312522- _wandbruntime2759  _runtime2759ʙ _runtime2759.082856095 _step492 & lr-AdamW/pg10.0001597617840224125 & lr-AdamW/pg20.0001597617840224125 trainer/global_step6199 % _timestamp1.7857980462598996e+09r _runtime2759.082856095 _step492 & lr-AdamW/pg10.0001597617840224125 & lr-AdamW/pg20.0001597617840224125 trainer/global_step6199 % _timestamp1.7857980462598996e+09OB2- _wandbruntime2759  _runtime27597"'O G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _step493 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed106438456290304 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed112292653714944 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 5602833797120 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 trainer/global_step6199 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated41191179 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed17753979 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated65478553 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated106438473329664 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5455204574208 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed42165774 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed111893573434880 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated111893677903872 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed41191177 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed41191177 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed109235379412884 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated24411797 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated112292807195648 # _timestamp1.78579804626083e+09 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 106660074 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated42166039 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed24411795 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated17754242 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5441766351412 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5455204574208 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 5455117144576 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current263 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated109235482255368 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5455117144576 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed103793613061472 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated106438473329664 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 106669732 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed65468897 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated65478553 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5602705220096 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0c\O A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5455117144576 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed103793613061472 Y̓y&KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated106438473329664 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 106669732 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed65468897 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated65478553 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5602705220096 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _step493 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed106438456290304 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed112292653714944 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 5602833797120 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 trainer/global_step6199 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated41191179 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed17753979 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated65478553 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated106438473329664 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5455204574208 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed42165774 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed111893573434880 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated111893677903872 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed41191177 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed41191177 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed109235379412884 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated24411797 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated112292807195648 # _timestamp1.78579804626083e+09 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 106660074 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated42166039 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed24411795 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated17754242 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5441766351412 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5455204574208 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 5455117144576 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current263 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated109235482255368 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251?a2- _wandbruntime2759  _runtime2759U))&M D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed65479420 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed112310759209984 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 trainer/global_step6199 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 5455996764160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed111911619817472 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5456084200960 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 % _timestamp1.7857980466409383e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed41197820 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated65489090 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated111911724293632 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated106707180515328 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed17756690 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 5603603598336 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated103810370273152 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 106686912 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed42172422 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed103810353233792 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current242 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 106677240 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 106686912 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated65489090 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130&M E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 106686912 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated65489090 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed65479420 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed112310759209984 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 trainer/global_step6199 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 5455996764160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed111911619817472 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5456084200960 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 % _timestamp1.7857980466409383e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed41197820 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated65489090 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated111911724293632 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated106707180515328 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed17756690 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 5603603598336 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated103810370273152 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 106686912 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed42172422 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed103810353233792 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current242 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 106677240 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 2537553922- _wandbruntime2759  _runtime2759BɨJ % _timestamp1.7857980466421795e+09 .loss/FBt2/detection_loss2.519472968764603 _runtime2759.465048567 _step495 /loss/FRt1/detection_loss2.3680527858231044 1loss/SRqqt1/detection_loss1.7960531411483203 &loss/total_loss21.515403206294817 /loss/FRt2/assignment_loss2.823513045426338 2loss/SRqqt1/assignment_loss1.2464034152030945 /loss/FBt1/assignment_loss0.114798037481259 0loss/FBt2/assignment_loss0.6578663064609165 /loss/FRt1/assignment_loss3.799004596307737 2loss/SRqqt2/assignment_loss1.5230781690212294 .loss/FRt2/detection_loss2.599671465254599 1loss/SRqqt2/detection_loss1.0153455544936623 /loss/FBt1/detection_loss1.0521437209099531 epoch0 trainer/global_step6199X17 /loss/FRt2/assignment_loss2.823513045426338 2loss/SRqqt1/assignment_loss1.2464034152030945 /loss/FBt1/assignment_loss0.114798037481259 0loss/FBt2/assignment_loss0.6578663064609165 /loss/FRt1/assignment_loss3.799004596307737 2loss/SRqqt2/assignment_loss1.5230781690212294 .loss/FRt2/detection_loss2.599671465254599 1loss/SRqqt2/detection_loss1.0153455544936623 /loss/FBt1/detection_loss1.0521437209099531 epoch0 trainer/global_step6199 % _timestamp1.7857980466421795e+09 .loss/FBt2/detection_loss2.519472968764603 _runtime2759.465048567 _step495 /loss/FRt1/detection_loss2.3680527858231044 1loss/SRqqt1/detection_loss1.7960531411483203 &loss/total_loss21.515403206294817Z(: U gpu.0.gpu41.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts98.308' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent32.769333333333336 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0aa̅(:  U disk.drbd1510.in0 disk.drbd2078.out0 disk.drbd1817.out0 disk.drbd1289.out0 disk.rbd0.out0* proc.memory.percent17.182016372680664$ disk./.usageGB578.3433303833008 disk.rbd1.in0 disk.drbd1552.in0 disk.drbd1552.out0 disk.drbd1285.in0 disk.drbd1289.in0 disk.sda1.out0 disk.dm-0.in 2.47265625! disk.dm-0.out111.87548828125 disk.drbd2118.in0 disk.drbd2111.out0 disk.drbd1965.out0 disk.drbd1828.in0 disk.drbd1226.out0 network.recv380304 disk.drbd1356.out0 disk.sda.out 75.078125 disk.drbd1510.out0$ memory_percent41.23557209968567 disk.drbd1550.in0 disk.rbd0.in0 disk.drbd2041.in0) disk./.usagePercent32.34398663433307 disk.drbd1356.in0 disk.sda2.out 75.078125 disk.drbd1931.out0 disk.drbd2041.ou=t0 disk.rbd1.out 11.05078125 disk.drbd1965.in0 disk.drbd1226.in0$ disk.nvme0n1.out111.87548828125 disk.drbd2111.in0 disk.drbd1673.in0 disk.drbd1938.in0 disk.drbd1938.out0 disk.drbd2078.in0 disk.drbd1584.in0* proc.memory.availableMB38511.85546875 disk.drbd1817.in0 disk.drbd1673.out0 disk.drbd1931.in0 disk.drbd1828.out0 cpu13.87907530741787 proc.cpu.threads29 disk.nvme0n1.in 2.47265625 disk.sda.in0! proc.memory.rssMB 11260.40625 network.sent 5.666146e+06 disk.drbd2118.out0 disk.sda2.in0 disk.drbd1550.out0 disk.drbd1285.out0 disk.drbd1584.out0 disk.sda1.in0RG(: U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts96.857' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent32.28566666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0= T(:  U disk.drbd1226.in0 disk.drbd2118.out0 disk.drbd1828.in0 disk.sda1.out0) disk./.usagePercent32.34378951016812 disk.drbd1510.out0 disk.nvme0n1.in 2.47265625 disk.drbd1931.out0 disk.drbd1285.in0 disk.drbd1550.out0! proc.memory.rssMB 11260.40625 disk.drbd1965.in0 network.recv381888 disk.drbd1584.out0 disk.drbd2118.in0 disk.drbd1289.out0 disk.drbd1817.out0 disk.drbd1510.in0 disk.dm-0.in 2.47265625 disk.drbd1965.out0 disk.drbd1552.out0 disk.rbd0.in0 network.sent 5.708892e+06 disk.drbd1931.in0 disk.drbd1673.in0 disk.drbd2041.in0* proc.memory.availableMB38512.06640625 cpu13.877341123963669 disk.rbd1.in0 disk.sda2.in0* proc.memory.percent17.182016372680664 disk.drbd1285.out0 disk.drbd2111.in0 disk.drbd1817.in0 disk.drbd1828.out0 disk.drbd1550.in0 disk.drbd1938.out0 disk.sda.out 75.546875 disk.sda1.in0$ memory_percent41.23525023460388$ disk./.usageGB578.3398056030273 disk.sda2.out 75.546875 disk.drbd1938.in0 disk.sda.in0 disk.drbd2111.out0 disk.rbd1.out 11.11328125$ disk.nvme0n1.out112.41552734375 disk.drbd2078.out0 disk.drbd1356.in0 disk.drbd1356.out0 disk.drbd1673.out0 disk.drbd1226.out0 disk.drbd2078.in0 disk.drbd1584.in0 disk.drbd2041.out0 disk.drbd1289.in0 disk.rbd0.out0! disk.dm-0.out112.41552734375 disk.drbd1552.in0 proc.cpu.threads292- _wandbruntime2781  _runtime2781nD & lr-AdamW/pg10.0001610503933466778 & lr-AdamW/pg20.0001610503933466778 trainer/global_step6249 % _timestamp1.7857980682714152e+09 _runtime2781.094597672 _step496䰎Z & lr-AdamW/pg20.0001610503933466778 trainer/global_step6249 % _timestamp1.7857980682714152e+09 _runtime2781.094597672 _step496 & lr-AdamW/pg10.0001610503933466778c=YT2- _wandbruntime2781  _runtime2781$'O E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed42505222 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed107550304334848 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed41523327 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated112796012189696 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated17896820 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 % _timestamp1.7857980682727945e+09 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5499200709632 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed112795907720704 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 trainer/global_step6249 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 107520226 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated112796012189696 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed112795907720704 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated107296811480064 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated113198306020352 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5647839816192 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed107296794440704 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated24608647 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed107296794440704 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed113198144151040 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5485654814920 _runtime 2781.10189917 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated110116379334876 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated66006555 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed65996899 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 5647976781824 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated104630638716832 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated42505467 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current245 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed17896577 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 _step497 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed65996899 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated41523329 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed41523327 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated107550329238528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current243 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated112796012189696 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed112795907720704 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated107296811480064 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated113198306020352 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5647839816192 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed107296794440704 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated24608647 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed107296794440704 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed113198144151040 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5485654814920 _runtime 2781.10189917 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated110116379334876 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated66006555 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed65996899 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 5647976781824 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated104630638716832 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated42505467 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current245 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed17896577 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 _step497 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed65996899 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated41523329 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed41523327 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated107550329238528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current243 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed104647361849792 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5486618091378 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 5486532288142 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated112814058579456 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated41529972 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed41529970 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated66017092 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated24612584 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 5648739931136 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5500080336384 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 _runtime2781.478662069 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed41529970 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed17899434 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed113216251382784 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 % _timestamp1.7857980686552613e+09 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 107547064 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 107537392 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed24612582 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 163959296 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed107313961203712 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed42512016 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated113216415342080 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated107313978243072 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5500080336384 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed107313961203712 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 5499992899584 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated66017092 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current266 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 107537392 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5648878986752 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated17899700 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated104647378889152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current268 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 _step498 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 107547064 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 trainer/global_step6249 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed112813954103296 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 % _timestamp1.7857980686552613e+09 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 107547064 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 107537392 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed24612582 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 163959296 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed107313961203712 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed42512016 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated113216415342080 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated107313978243072 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5500080336384 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed107313961203712 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 5499992899584 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated66017092 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current266 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 107537392 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5648878986752 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated17899700 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated104647378889152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current268 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 _step498 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 107547064 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 trainer/global_step6249 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed112813954103296 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed104647361849792 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5486618091378 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 5486532288142 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated112814058579456 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated41529972 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed41529970 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated66017092 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated24612584 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 5648739931136 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5500080336384 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888jB2- _wandbruntime2781  _runtime2781Ă epoch0 1loss/SRqqt2/detection_loss1.1882145057855162 -loss/FBt1/detection_loss1.00932729189434 _step499 trainer/global_step6249 % _timestamp1.7857980686571522e+09 /loss/FBt2/detection_loss2.5010764258248463 %loss/total_loss21.17615479305274 0loss/FRt1/assignment_loss3.9119223413559108 0loss/FRt2/assignment_loss2.6924798827545318 2loss/SRqqt2/assignment_loss1.4896239101018276 1loss/FBt1/assignment_loss0.11761165711338109 0loss/FBt2/assignment_loss0.1617096915931922 1loss/SRqqt1/detection_loss1.9873001678358941 2loss/SRqqt1/assignment_loss1.1011093911670504 .loss/FRt1/detection_loss2.373544292954298 .loss/FRt2/detection_loss2.642235234671948 _runtime2781.480029713Jb /loss/FBt2/detection_loss2.5010764258248463 %loss/total_loss21.17615479305274 0loss/FRt1/assignment_loss3.9119223413559108 0loss/FRt2/assignment_loss2.6924798827545318 2loss/SRqqt2/assignment_loss1.4896239101018276 1loss/FBt1/assignment_loss0.11761165711338109 0loss/FBt2/assignment_loss0.1617096915931922 1loss/SRqqt1/detection_loss1.9873001678358941 2loss/SRqqt1/assignment_loss1.1011093911670504 .loss/FRt1/detection_loss2.373544292954298 .loss/FRt2/detection_loss2.642235234671948 _runtime2781.480029713 epoch0 1loss/SRqqt2/detection_loss1.1882145057855162 -loss/FBt1/detection_loss1.00932729189434 _step499 trainer/global_step6249 % _timestamp1.7857980686571522e+09;W(: ʒU gpu.0.gpu24.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts96.052' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent32.01733333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0ɘR(:  V disk.drbd1510.out0! disk.dm-0.out114.02587890625 disk.drbd1817.out0 disk.drbd1931.in0 disk.drbd1552.in0 disk.drbd1285.in0 disk.drbd1226.out0$ disk./.usageGB578.3383598327637 disk.drbd1584.in0 disk.drbd1938.in0 disk.drbd1673.in0 disk.drbd1552.out0 disk.drbd1510.in0 disk.drbd1673.out0 disk.drbd2078.in0 disk.drbd2041.in0 disk.nvme0n1.in 2.47265625 disk.drbd1550.in0 disk.drbd1289.in0! proc.memory.rssMB 11260.40625 disk.drbd2111.out0 disk.drbd1965.out0 disk.drbd1931.out0 disk.drbd1938.out0 proc.cpu.threads29 network.recv384387 disk.drbd2111.in0 disk.drbd1965.in0 disk.drbd1550.out0 disk.drbd1356.out0 disk.sda.out 75.9140625 disk.sda1.in0% memory_percent41.234540939331055 network.sent 5.7562e+06) disk./.usagePercent32.34370865512644 cpu13.877585002289912 disk.drbd1356.in0 disk.sda2.in0 disk.sda2.out 75.9140625 disk.drbd1289.out0 disk.dm-0.in 2.47265625' proc.memory.availableMB 38512.53125 disk.drbd1584.out0 disk.drbd2118.in0 disk.drbd2118.out0 disk.drbd1817.in0 disk.rbd1.out 11.11328125 disk.drbd1828.out0 disk.rbd0.in0 disk.rbd1.in0 disk.drbd2041.out0 disk.sda.in0 disk.sda1.out0 disk.drbd1285.out0 disk.drbd1226.in0* proc.memory.percent17.182016372680664 disk.drbd1828.in0 disk.drbd2078.out0$ disk.nvme0n1.out114.02587890625 disk.rbd0.out0U"2- _wandbruntime2803  _runtime2803V) & lr-AdamW/pg10.0001623390026709431 & lr-AdamW/pg20.0001623390026709431 trainer/global_step6299 % _timestamp1.7857980910215037e+09 _runtime 2803.84447771 _step500^b % _timestamp1.7857980910215037e+09 _runtime 2803.84447771 _step500 & lr-AdamW/pg10.0001623390026709431 & lr-AdamW/pg20.0001623390026709431 trainer/global_step6299a2- _wandbruntime2803  _runtime2803/0'O WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5543196845056 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated113698346475520 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed18040506 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated110997276414384 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed66524901 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5543109415424 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated108410685078528 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 108390036 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed41855477 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated41855479 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed113698242006528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 108380378 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed41855477 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated42846255 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed24805495 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed108410660174848 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated66534557 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated108155149630464 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed108155132591104 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed42846001 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated41855479 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 _step501 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated18040758 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed105467630293472 _runtime 2803.84584402 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated105467647332832 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed108155132591104 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated108155149630464 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 5543109415424 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5543196845056 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated113698346475520 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5529543278428 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed66524901 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated66534557 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current254 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated24805497 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 5529629081552 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 trainer/global_step6299 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed113698242006528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0g"O JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current254 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated24805497 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 5529629081552 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 trainer/global_step6299 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed113698242006528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5543196845056 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated113698346475520 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed18040506 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated110997276414384 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed66524901 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5543109415424 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated108410685078528 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 108390036 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed41855477 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated41855479 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed113698242006528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 108380378 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed41855477 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated42846255 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed24805495 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed108410660174848 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated66534557 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated108155149630464 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed108155132591104 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed42846001 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated41855479 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 _step501 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated18040758 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed105467630293472 _runtime 2803.84584402 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated105467647332832 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed108155132591104 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 MGDeviceStats_ Monitor.on_train_batch_start/reserved_bytes.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated108155149630464 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 5543109415424 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5543196845056 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated113698346475520 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5529543278428 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed66524901 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated66534557%2- _wandbruntime2804  _runtime2804"m,2&M TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5544076471808 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated111014894060038 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed108427867291648 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed111014791217442 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated113716392865280 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5543989035008 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated41862122 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated66545094 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated113716392865280 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed18043295 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated105484387505152 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 % _timestamp1.7857980914019718e+09 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 108397544 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 5693893377024 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated24809434 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed42852727 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5694026141184 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated66545094 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed105484370465792 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5530506554886 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed113716288389120 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed108172299354112 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 108407216 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed66535424 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated41862122 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated108172316393472 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 _runtime 2804.22537936 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 trainer/global_step6299 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed66535424 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated42852996 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 108407216 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 _step502 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated18043562 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated114121918336512 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed108172299354112&M J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed41862120 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated108172316393472 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed42852727 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5694026141184 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated66545094 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed105484370465792 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5530506554886 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed113716288389120 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed108172299354112 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 108407216 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed66535424 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated41862122 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated108172316393472 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 _runtime 2804.22537936 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 trainer/global_step6299 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed66535424 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated42852996 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 108407216 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 _step502 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated18043562 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated114121918336512 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed108172299354112 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5544076471808 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated111014894060038 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed108427867291648 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed111014791217442 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated113716392865280 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5543989035008 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated41862122 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated66545094 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated113716392865280 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed18043295 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated105484387505152 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 % _timestamp1.7857980914019718e+09 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 108397544 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 5693893377024 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated24809434 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 PDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed25002345 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed114600576292352 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5587105550848 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed111878070651408 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 _step505 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed42187627 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current236 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed67052903 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed109013470741504 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed109271016014848 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated109271040918528 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5587192980480 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 109240530 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated67062559 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated106304655948832 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated109013487780864 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated42187629 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated25002347 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 5738329067520 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5738202587648 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated18182242 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated42187629 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed42187627 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated111878173493892 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed109013470741504 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 126479872 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed106304638909472 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed67052903 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed115009218602496 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current96582j'O G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5587192980480 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 109240530 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated67062559 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated106304655948832 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated109013487780864 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated42187629 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated25002347 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 5738329067520 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5738202587648 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated18182242 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated42187629 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed42187627 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated111878173493892 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed109013470741504 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 126479872 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed106304638909472 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed67052903 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed115009218602496 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed25002345 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed114600576292352 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5587105550848 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed111878070651408 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 _step505 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed42187627 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current236 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed67052903 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed109013470741504 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed109271016014848 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated109271040918528 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 02- _wandbruntime2826  _runtime2826Tg`&M =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed109030637504512 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 5587985170432 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated25006284 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated109288248035328 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed111895688296950 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed106321379081792 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 109267368 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated114618727151104 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated111895791139546 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated42194272 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5588072607232 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed25006282 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed115027328308224 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed42194270 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current253 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated115027485976064 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 trainer/global_step6349 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 5739105176576 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed18184837 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed109030637504512 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5587985170432 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5588072607232 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5574395018394 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated67073096 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed114618622674944 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _runtime2826.452774308 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 $ _timestamp1.785798113629374e+09 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated109030654543872 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated42194272 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed109288223131648 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated109030654543872 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 109257696 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 109257696 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated43191372 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed43191119 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated18185088 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated106321396121152 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 _step506 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed43191119 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated18185088 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated106321396121152 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 _step506 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 5587985170432 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated25006284 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated109288248035328 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed111895688296950 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed106321379081792 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 109267368 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated114618727151104 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated111895791139546 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated42194272 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5588072607232 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed25006282 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed115027328308224 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed42194270 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current253 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated115027485976064 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 trainer/global_step6349 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 5739105176576 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed18184837 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed109030637504512 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5587985170432 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5588072607232 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5574395018394 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated67073096 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed114618622674944 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _runtime2826.452774308 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 $ _timestamp1.785798113629374e+09 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated109030654543872!2- _wandbruntime2826  _runtime28266׸ 2loss/SRqqt1/assignment_loss1.3661044946423284 1loss/SRqqt2/assignment_loss1.309350093025402 1loss/SRqqt2/detection_loss0.9934313789467905 .loss/FBt1/detection_loss1.282378189681497 _runtime2826.454082375 1loss/SRqqt1/detection_loss1.8754183346474613 0loss/FRt2/assignment_loss2.4640214840571084 2loss/FBt1/assignment_loss0.010586137618758025 /loss/FRt2/detection_loss2.6752233018004703 &loss/total_loss26.488296937159397 epoch0 trainer/global_step6349 % _timestamp1.7857981136312299e+09 /loss/FRt1/assignment_loss3.684828001673859 2loss/FBt2/assignment_loss0.002705987310033464 .loss/FRt1/detection_loss2.472479609569701 .loss/FBt2/detection_loss8.351769924185986 _step507GƏ  2loss/SRqqt1/assignment_loss1.3661044946423284 1loss/SRqqt2/assignment_loss1.309350093025402 1loss/SRqqt2/detection_loss0.9934313789467905 .loss/FBt1/detection_loss1.282378189681497 _runtime2826.454082375 1loss/SRqqt1/detection_loss1.8754183346474613 0loss/FRt2/assignment_loss2.4640214840571084 2loss/FBt1/assignment_loss0.010586137618758025 /loss/FRt2/detection_loss2.6752233018004703 &loss/total_loss26.488296937159397 epoch0 trainer/global_step6349 % _timestamp1.7857981136312299e+09 /loss/FRt1/assignment_loss3.684828001673859 2loss/FBt2/assignment_loss0.002705987310033464 .loss/FRt1/detection_loss2.472479609569701 .loss/FBt2/detection_loss8.351769924185986 _step507z!l(: РU gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts124.438' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent41.47933333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0MS@R(:  U disk.drbd1552.out0 disk.drbd1828.in0 disk.sda.out 77.0546875 disk.drbd1584.in0 disk.sda2.in0# disk.nvme0n1.out117.5322265625& proc.memory.availableMB 38511.9375) disk./.usagePercent32.34421341245789 disk.drbd2041.in0 disk.drbd1289.out0 disk.sda1.in0 disk.drbd1226.in0 cpu13.879012776295736 disk.drbd1552.in0 disk.drbd1550.in0 disk.rbd0.in0 disk.sda.in0 disk.drbd2118.out0 disk.dm-0.in 2.47265625 disk.drbd2118.in0 proc.cpu.threads29 disk.sda1.out0 disk.drbd2078.out0 network.sent 5.807065e+06 disk.drbd1938.in0 disk.nvme0n1.in 2.47265625 disk.drbd1965.out0 disk.drbd2041.out0 disk.drbd1285.in0 disk.drbd1285.out0 disk.drbd1817.in0 disk.rbd1.out 11.3876953125 disk.drbd1965.in0 disk.drbd1550.out0 disk.drbd1289.in0 disk.drbd1584.out0 disk.drbd1356.out0 disk.drbd1931.out0 disk.dm-0.out117.5322265625 disk.drbd1226.out0$ disk./.usageGB578.3473854064941 disk.drbd1938.out0$ memory_percent41.23544692993164 network.recv387686 disk.drbd1673.out0 disk.rbd0.out0 disk.drbd1510.out0 disk.drbd1356.in0 disk.sda2.out 77.0546875 disk.drbd1817.out0 disk.drbd1510.in0 disk.drbd2078.in0 disk.drbd2111.out0! proc.memory.rssMB 11260.40625* proc.memory.percent17.182016372680664 disk.drbd1828.out0 disk.drbd1673.in0 disk.drbd1931.in0 disk.drbd2111.in0 disk.rbd1.in0|2- _wandbruntime2848  _runtime2848,+N & lr-AdamW/pg10.0001649162213194737 & lr-AdamW/pg20.0001649162213194737 trainer/global_step6399 % _timestamp1.7857981355802016e+09 _runtime2848.404700685 _step508d & lr-AdamW/pg10.0001649162213194737 & lr-AdamW/pg20.0001649162213194737 trainer/global_step6399 % _timestamp1.7857981355802016e+09 _runtime2848.404700685 _step508c2- _wandbruntime2848  _runtime2848M;J'O L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed42519777 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated112759070573400 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current267 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed115914725084160 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed18324509 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed67580905 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5617320205444 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 _runtime2848.404824246 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated109871825931264 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5631189115904 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 trainer/global_step6399 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated67590561 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 5617406008568 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed109871808891904 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed109871808891904 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated43523971 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated107141664564832 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated67590561 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5631101686272 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 110110340 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated42519779 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated110131396758528 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 5631101686272 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed110131371854848 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed42519777 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed115502910578176 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current265 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed107141647525472 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed43523704 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed115502910578176 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5631189115904 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated25199197 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated115503015047168 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 5783481806336 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992L'O UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed112758967730916 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed67580905 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 110100682 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 $ _timestamp1.785798135580896e+09 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5783353229312 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 _step509 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated115503015047168 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 5631101686272 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed110131371854848 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed42519777 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed115502910578176 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current265 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed107141647525472 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed43523704 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed115502910578176 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5631189115904 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated25199197 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated115503015047168 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 5783481806336 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed42519777 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated112759070573400 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current267 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed115914725084160 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed18324509 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed67580905 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5617320205444 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 _runtime2848.404824246 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated109871825931264 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5631189115904 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 trainer/global_step6399 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated67590561 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 5617406008568 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed109871808891904 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed109871808891904 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated43523971 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated107141664564832 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated67590561 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5631101686272 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 110110340 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated42519779 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated110131396758528 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed43530473 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed42526420 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5631981305856 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current256 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed107158387697792 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 5618197678666 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed109888975654912 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 5631981305856 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated107158404737152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5632068742656 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed25203132 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed112776585376458 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated109888992694272 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated18327595 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 % _timestamp1.7857981359622362e+09 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5784388959232 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed42526420 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.cuKyrrent254 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 _runtime2848.785618683 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed67591428 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 _step510 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated25203134 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed18327341 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 5784254097920 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated109888992694272 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 trainer/global_step6399 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed115932833069568 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated42526422 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5632068742656 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated115521061436928 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 110127520 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated42526422 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated112776688219054 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032jʳ&M NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed67591428 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 _step510 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated25203134 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed18327341 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 5784254097920 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated109888992694272 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 trainer/global_step6399 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed115932833069568 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated42526422 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5632068742656 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated115521061436928 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 110127520 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated42526422 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated112776688219054 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed43530473 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed42526420 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5631981305856 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current256 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed107158387697792 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 5618197678666 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed109888975654912 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 5631981305856 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated107158404737152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5632068742656 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed25203132 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed112776585376458 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated109888992694272 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated18327595 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 % _timestamp1.7857981359622362e+09 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5784388959232 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed42526420 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current254 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 _runtime2848.785618683aI2- _wandbruntime2848  _runtime2848` 2loss/SRqqt1/assignment_loss1.5112910358344807 1loss/SRqqt1/detection_loss1.8791843045722036 .loss/FBt2/detection_loss7.540318727493286 /loss/FRt2/assignment_loss2.460012221491182 1loss/FBt1/assignment_loss0.06624779113323191 /loss/FBt1/detection_loss1.6798140611790586 %loss/total_loss27.10705056870269 epoch0 % _timestamp1.7857981359639606e+09 /loss/FRt1/assignment_loss4.021453783625648 2loss/SRqqt2/assignment_loss1.4114623197132128 .loss/FRt1/detection_loss2.649182174886976 /loss/FRt2/detection_loss2.2836568719380863 trainer/global_step6399 _runtime2848.786804784 0loss/FBt2/assignment_loss0.5109202265739157 1loss/SRqqt2/detection_loss1.0935070502614095 _step511j} 2loss/SRqqt1/assignment_loss1.5112910358344807 1loss/SRqqt1/detection_loss1.8791843045722036 .loss/FBt2/detection_loss7.540318727493286 /loss/FRt2/assignment_loss2.460012221491182 1loss/FBt1/assignment_loss0.06624779113323191 /loss/FBt1/detection_loss1.6798140611790586 %loss/total_loss27.10705056870269 epoch0 % _timestamp1.7857981359639606e+09 /loss/FRt1/assignment_loss4.021453783625648 2loss/SRqqt2/assignment_loss1.4114623197132128 .loss/FRt1/detection_loss2.649182174886976 /loss/FRt2/detection_loss2.2836568719380863 trainer/global_step6399 _runtime2848.786804784 0loss/FBt2/assignment_loss0.5109202265739157 1loss/SRqqt2/detection_loss1.0935070502614095 _step511(: ݙU gpu.0.gpu23.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp40.0 gpu.0.powerWatts127.572' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent42.524 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0SV(:  U disk.drbd1285.in0 disk.drbd1226.in0 disk.drbd1356.out0 disk.drbd2111.in0 disk.drbd1817.out0 disk.rbd1.in0 network.sent 5.852205e+06 disk.drbd1673.in0 disk.drbd1552.in0 disk.drbd1289.in0 disk.rbd0.out0 disk.sda.out 77.296875 disk.drbd1510.in0 disk.dm-0.in 2.47265625 disk.drbd1584.in0 disk.drbd1817.in0 disk.drbd1931.out0 disk.drbd1828.out0 disk.drbd1552.out0$ memory_percent41.23550057411194 proc.cpu.threads29) disk./.usagePercent32.34476019734399 disk.sda2.out 77.296875 disk.drbd1965.in0 disk.sda2.in0 disk.drbd1673.out0 disk.drbd1938.in0* proc.memory.percent17.182016372680664 cpu13.877150778311377 disk.drbd1226.out0 disk.drbd2118.in0 disk.drbd1828.in0 disk.sda1.out0 disk.drbd2078.in0 disk.nvme0n1.in 2.47265625* proc.memory.availableMB38511.90234375 network.recv390071 disk.sda1.in0 disk.drbd2118.out0 disk.drbd1550.out0 disk.drbd1550.in0 disk.drbd1289.out0 disk.rbd0.in0 disk.sda.in0 disk.drbd2111.out0 disk.rbd1.out 11.3876953125 disk.drbd1285.out0 disk.drbd1510.out0! disk.dm-0.out119.35205078125 disk.drbd1584.out0 disk.drbd1356.in0$ disk./.usageGB578.3571624755859 disk.drbd1965.out0 disk.drbd2041.in0 disk.drbd1938.out0 disk.drbd2078.out0$ disk.nvme0n1.out119.35205078125 disk.drbd1931.in0 disk.drbd2041.out0! proc.memory.rssMB 11260.40625仛(: łU gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts128.449' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.81633333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0CW(:  V network.recv392375 disk.nvme0n1.in 2.47265625 disk.drbd1673.out0 disk.drbd2041.in0 disk.drbd1510.in0 disk.drbd2111.out0 disk.drbd1817.in0 disk.drbd1817.out0 disk.drbd1289.out0! disk.dm-0.out123.09814453125 disk.drbd1931.in0 disk.drbd1828.in0 disk.drbd2078.in0 disk.drbd2118.out0 disk.sda2.in0 disk.drbd1584.in0 disk.drbd1584.out0 disk.dm-0.in 2.47265625 disk.drbd2111.in0 disk.drbd1938.out0 disk.drbd1226.in0 disk.sda2.out 77.7265625 disk.rbd1.out 11.419921875 disk.sda1.in0 disk.drbd1510.out0 disk.drbd2078.out0 disk.drbd1356.in0 disk.drbd1965.out0 disk.drbd1552.in0 disk.drbd1828.out0 disk.drbd1550.out0 disk.drbd1938.in0 disk.drbd1931.out0 disk.drbd2041.out0 disk.rbd0.in0 disk.sda1.out0 disk.drbd1285.out0 disk.drbd2118.in0 disk.rbd0.out0 disk.sda.out 77.7265625 network.sent 5.897533e+06$ disk.nvme0n1.out123.09814453125 disk.drbd1356.out0 disk.rbd1.in0$ memory_percent41.23550057411194 cpu13.876369684041652 disk.drbd1226.out0* proc.memory.availableMB38511.90234375! proc.memory.rssMB 11260.40625 disk.drbd1965.in0 disk.sda.in0$ disk./.usageGB578.3566474914551 disk.drbd1673.in0 disk.drbd1552.out0 disk.drbd1550.in0 proc.cpu.threads29) disk./.usagePercent32.34473139673547 disk.drbd1289.in0 disk.drbd1285.in0* proc.memory.percent17.182016372680664/F2- _wandbruntime2870  _runtime2870ŝD ' lr-AdamW/pg20.00016620483064373902 trainer/global_step6449 % _timestamp1.7857981579623935e+09 _runtime2870.785372216 _step512 ' lr-AdamW/pg10.00016620483064373902 _step512 ' lr-AdamW/pg10.00016620483064373902 ' lr-AdamW/pg20.00016620483064373902 trainer/global_step6449 % _timestamp1.7857981579623935e+09 _runtime2870.785372216K\82- _wandbruntime2870  _runtime2870jne'O MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed116405244864000 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated107978673180832 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5675185251328 trainer/global_step6449 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated116405349332992 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated116405349332992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed110730147042304 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 110970492 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed116405244864000 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated110991752598528 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed68108907 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated113639967652908 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed113639864810424 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _runtime2870.786625688 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated25396047 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 5661294472076 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 _step513 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated68118563 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed68108907 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated116820398282752 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5675185251328 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024mO QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed107978656141472 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated113639967652908 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed113639864810424 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 G?DeviceStat%sMonitor.on_train_batch_start/segment.small_pool.peak121 _runtime2870.786625688 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated25396047 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 5661294472076 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 _step513 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated68118563 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed68108907 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated116820398282752 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5675185251328 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed116405244864000 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated107978673180832 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5675185251328 trainer/global_step6449 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated116405349332992 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated116405349332992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed110730147042304 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 110970492 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed116405244864000 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated110991752598528 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed68108907 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 JDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 110978000 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 % _timestamp1.7857981583320181e+09 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 5662086142174 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed68119430 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated110747330844672 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated43871128 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed42858570 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated116423395722752 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed111008934811648 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated107995413353152 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed116423291246592 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated42858572 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5662171945410 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed18470895 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 5675977441280 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated68129100 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 110987672 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed42858570 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5829563593216 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed110747313805312 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed68119430 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed43870877 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated42858572 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated110747330844672 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current249 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated68129100 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated116423395722752 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5676064878080 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed116838365640704 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated111008959715328 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated18471144 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed113657482455966 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated1305N&M E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 110978000 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 % _timestamp1.7857981583320181e+09 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 5662086142174 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed68119430 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated110747330844672 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated43871128 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed42858570 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated116423395722752 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed111008934811648 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated107995413353152 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed116423291246592 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated42858572 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5662171945410 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed18470895 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 5675977441280 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated68129100 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 110987672 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed42858570 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5829563593216 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed110747313805312 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed68119430 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed43870877 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated42858572 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated110747330844672 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current249 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated68129100 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated116423395722752 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5676064878080 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed116838365640704 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated111008959715328 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated18471144 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed113657482455966 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated116838523308544 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed107995396313792 trainer/global_step6449 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed25399982 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 5829430829056 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121$<2- _wandbruntime2871  _runtime2871¸ &loss/total_loss25.246768590853016 0loss/FRt1/assignment_loss3.8184158418859755 1loss/FBt2/assignment_loss0.49107132851372626 .loss/FRt2/detection_loss2.548971234569474 _runtime2871.156711619 /loss/FRt1/detection_loss2.3164578724120344 _step515 epoch0 /loss/FRt2/assignment_loss2.586763867310115 2loss/SRqqt2/assignment_loss1.4880003979506753 1loss/SRqqt1/detection_loss1.8896090004480246 1loss/SRqqt2/detection_loss0.9731802547119073 /loss/FBt2/detection_loss0.8447246477007866 trainer/global_step6449 $ _timestamp1.785798158333862e+09 2loss/SRqqt1/assignment_loss1.3104120955774279 0loss/FBt1/assignment_loss0.0403790965662971 -loss/FBt1/detection_loss6.93878295320657˽R 0loss/FRt1/assignment_loss3.8184158418859755 1loss/FBt2/assignment_loss0.49107132851372626 .loss/FRt2/detection_loss2.548971234569474 _runtime2871.156711619 /loss/FRt1/detection_loss2.3164578724120344 _step515 epoch0 /loss/FRt2/assignment_loss2.586763867310115 2loss/SRqqt2/assignment_loss1.4880003979506753 1loss/SRqqt1/detection_loss1.8896090004480246 1loss/SRqqt2/detection_loss0.9731802547119073 /loss/FBt2/detection_loss0.8447246477007866 trainer/global_step6449 $ _timestamp1.785798158333862e+09 2loss/SRqqt1/assignment_loss1.3104120955774279 0loss/FBt1/assignment_loss0.0403790965662971 -loss/FBt1/detection_loss6.93878295320657 &loss/total_loss25.246768590853016fO(: U gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts125.318' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent41.772666666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0q|!(:  V disk.drbd1510.out0 disk.drbd1673.in0 disk.drbd1931.out0 network.sent 5.900551e+06 disk.drbd2078.out0 disk.sda2.out 78.30078125 disk.drbd2111.out0 disk.drbd1828.out0 disk.drbd2118.in0 disk.sda2.in0 disk.drbd1817.in0 disk.drbd2041.in0 disk.drbd1550.in0$ memory_percent41.23556613922119 disk.drbd1965.in0 disk.rbd0.in0 disk.drbd1285.out0 disk.drbd1584.out0 disk.drbd2041.out0 disk.drbd1828.in0 disk.sda.out 78.30078125 disk.drbd1226.in0 cpu13.878101448088614 disk.drbd2111.in0 disk.drbd1673.out0 disk.dm-0.in 2.47265625( proc.memory.availableMB 38511.859375 disk.rbd1.out 11.513671875 disk.drbd1931.in0 disk.drbd1552.in0 disk.rbd0.out0! proc.memory.rssMB 11260.40625( disk./.usagePercent32.3442003988496 disk.nvme0n1.in 2.47265625$ disk.nvme0n1.out134.05029296875! disk.dm-0.out134.05029296875 disk.drbd2078.in0 disk.drbd1817.out0 disk.drbd1356.in0 disk.drbd1965.out0 disk.drbd1289.out0* proc.memory.percent17.182016372680664 network.recv393378$ disk./.usageGB578.3471527099609 -sdisk.drbd1584.in0 disk.drbd1356.out0 disk.rbd1.in0 disk.drbd1552.out0 disk.drbd1550.out0 disk.drbd1938.in0 disk.drbd1938.out0 disk.drbd1285.in0 disk.drbd1226.out0 disk.sda.in0 disk.drbd2118.out0 disk.sda1.in0 disk.sda1.out0 disk.drbd1510.in0 proc.cpu.threads29 disk.drbd1289.in0&(2- _wandbruntime2893  _runtime2893 _step516 ' lr-AdamW/pg10.00016749343996800433 ' lr-AdamW/pg20.00016749343996800433 trainer/global_step6499 % _timestamp1.7857981804894526e+09 _runtime2893.312428813}s$ ' lr-AdamW/pg10.00016749343996800433 ' lr-AdamW/pg20.00016749343996800433 trainer/global_step6499 % _timestamp1.7857981804894526e+09 _runtime2893.312428813 _step5162- _wandbruntime2893  _runtime2893H'O L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed43184077 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated117307683618816 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed68636909 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated25592897 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed68636909 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed18611354 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated114520864732416 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated117725965246464 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed111588485192704 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 $ _timestamp1.785798180490431e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated68646565 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed117307579149824 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated111588502232064 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 5705182935584 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 _step517 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated44204514 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated117307683618816 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated108815681796832 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed25592895 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed117725801280000 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed111588485192704 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5719093957120 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated111852108438528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed114520761889932 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 _runtime 2893.31377837 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5719181386752 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed111852083534848 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5873717745152 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 5873856807936 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated43184079 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 trainer/global_step6499 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current265 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated43184079 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed108815664757472 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed117307579149824 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated111588502232064p'O L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed43184077 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated117307683618816 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed68636909 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated25592897 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed68636909 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed18611354 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated114520864732416 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated117725965246464 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed111588485192704 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 $ _timestamp1.785798180490431e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated68646565 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed117307579149824 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated111588502232064 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 5705182935584 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 _step517 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated44204514 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated117307683618816 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated108815681796832 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed25592895 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed117725801280000 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed111588485192704 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5719093957120 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated111852108438528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed114520761889932 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 _runtime 2893.31377837 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5719181386752 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed111852083534848 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5873717745152 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 5873856807936 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated43184079 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 trainer/global_step6499 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current265 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated43184079 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed108815664757472 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed117307579149824 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated111588502232064ЫY2- _wandbruntime2893  _runtime2893&T&M O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 $ _timestamp1.785798180869123e+09 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5720061013504 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed68647432 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated111605668995072 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5874758472192 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated68657102 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed43190720 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated111605668995072 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed25596832 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated108832421969152 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated43190722 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated114538482378070 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated43190722 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated68657102 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed117325625532416 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed68647432 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 5719973576704 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current265 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 5874623610880 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5706060408918 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed114538379535474 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed108832404929792 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed18614354 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed117743914262528 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed44211186 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 111838152 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed43190720 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5720061013504 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated117325730008576 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 111847824 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated43190722 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated68657102 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed117325625532416 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed68647432 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 5719973576704 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current265 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 5874623610880 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5706060408918 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 OADeviceStatsMonitor.on_train_batch_en=F!d/allocated_bytes.all.current 104476160 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed114538379535474 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed108832404929792 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed18614354 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed117743914262528 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed44211186 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 111838152 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed43190720 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5720061013504 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated117325730008576 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 111847824 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 $ _timestamp1.785798180869123e+09 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5720061013504 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed68647432 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated111605668995072 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5874758472192 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated68657102 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed43190720 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated111605668995072 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed25596832 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated108832421969152 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated43190722 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated114538482378070 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680L2a2- _wandbruntime2893  _runtime2893?13 trainer/global_step6499 &loss/total_loss18.921121835301946 _runtime2893.693774129 2loss/SRqqt1/assignment_loss1.4771470609638426 2loss/SRqqt2/assignment_loss1.3824114770827334 0loss/FBt2/assignment_loss0.3164337817575529 0loss/SRqqt1/detection_loss2.085730954728745 _step519 /loss/FRt2/assignment_loss2.025064594036824 /loss/FRt1/detection_loss2.3755062088788113 .loss/FRt2/detection_loss2.366165550979408 1loss/SRqqt2/detection_loss0.9885405394656772 /loss/FBt1/detection_loss0.8496497094631195 /loss/FBt2/detection_loss1.0794329717755318 % _timestamp1.7857981808709075e+09 0loss/FRt1/assignment_loss3.9679365024388393 2loss/FBt1/assignment_loss0.007102483730859075 epoch0 =  &loss/total_loss18.921121835301946 _runtime2893.693774129 2loss/SRqqt1/assignment_loss1.4771470609638426 2loss/SRqqt2/assignment_loss1.3824114770827334 0loss/FBt2/assignment_loss0.3164337817575529 0loss/SRqqt1/detection_loss2.085730954728745 _step519 /loss/FRt2/assignment_loss2.025064594036824 /loss/FRt1/detection_loss2.3755062088788113 .loss/FRt2/detection_loss2.366165550979408 1loss/SRqqt2/detection_loss0.9885405394656772 /loss/FBt1/detection_loss0.8496497094631195 /loss/FBt2/detection_loss1.0794329717755318 % _timestamp1.7857981808709075e+09 0loss/FRt1/assignment_loss3.9679365024388393 2loss/FBt1/assignment_loss0.007102483730859075 epoch0 trainer/global_step6499(: U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts124.002' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent41.333999999999996 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0s=\J(:  V disk.drbd1938.out0 disk.nvme0n1.in 2.47265625 disk.drbd2078.out0 disk.drbd1584.out0 disk.nvme0n1.out134.9375 disk.sda1.out0 disk.sda.in0 disk.drbd1289.in0 disk.drbd1938.in0 disk.drbd2118.out0 disk.drbd1817.in0 disk.drbd1965.in0 disk.drbd1550.out0 disk.drbd1285.in0 disk.rbd1.out 11.576171875 cpu13.87892966311003) disk./.usagePercent32.34384775139868 disk.drbd1285.out0 disk.drbd1673.in0 disk.drbd2118.in0 disk.drbd1356.in0 disk.drbd1510.in0 disk.rbd1.in0* proc.memory.availableMB38512.54296875 disk.drbd1828.in0 disk.drbd1931.in0 disk.drbd1289.out0 disk.rbd0.in0 disk.sda1.in0! proc.memory.rssMB 11260.40625 disk.drbd1965.out0$ memory_percent41.23452305793762 disk.rbd0.out0 disk.drbd1356.out0* proc.memory.percent17.182016372680664 disk.drbd1552.in0 network.sent 5.945755e+06 network.recv395877 disk.drbd2041.in0 disk.drbd2041.out0 disk.drbd1226.out0$ disk./.usageGB578.3408470153809 disk.dm-0.in 2.47265625 disk.drbd1226.in0 disk.drbd2111.in0 disk.sda.out 78.61328125 disk.drbd1931.out0 disk.dm-0.out134.9375 disk.drbd1584.in0 disk.sda2.out 78.61328125 proc.cpu.threads29 disk.drbd1817.out0 disk.drbd1673.out0 disk.drbd1550.in0 disk.drbd1510.out0 disk.drbd2078.in0 disk.sda2.in0 disk.drbd2111.out0 disk.drbd1552.out0 disk.drbd1828.out0(#(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts96.469' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent32.15633333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0azIV(:  V network.recv397799# disk./.usageGB578.340892791748 disk.drbd2078.out0 disk.drbd1965.in0 disk.drbd2041.out0 disk.drbd1931.in0 disk.sda1.in0 disk.drbd1285.out0) proc.memory.availableMB 38512.0078125 disk.drbd2078.in0 disk.drbd1673.in0 disk.sda.out 78.9296875 cpu13.880338438924738% memory_percent41.235339641571045 disk.drbd1584.in0 disk.drbd2118.out0 disk.drbd1356.out0 disk.rbd1.in0 disk.drbd2118.in0* proc.memory.percent17.182016372680664 disk.nvme0n1.in 2.47265625) disk./.usagePercent32.34385031145276 disk.drbd1226.out0 disk.drbd2111.out0 disk.drbd1552.in0 disk.rbd0.in0! disk.dm-0.out136.44970703125 proc.cpu.threads29$ disk.nvme0n1.out136.44970703125 disk.drbd1552.out0 disk.drbd1289.out0 disk.drbd1584.out0 disk.drbd1356.in0 disk.sda2.in0 disk.drbd1226.in0 disk.drbd1817.in0 disk.drbd1673.out0 disk.drbd1828.out0 disk.drbd1938.out0 disk.rbd0.out0! proc.memory.rssMB 11260.40625 disk.drbd1510.out0 disk.sda2.out 78.9296875 disk.drbd2111.in0 disk.drbd1965.out0 disk.drbd2041.in0 disk.drbd1550.out0 disk.drbd1289.in0 disk.drbd1938.in0 disk.rbd1.out 11.576171875 disk.drbd1550.in0 disk.sda1.out0 disk.drbd1510.in0 network.sent 5.991009e+06 disk.drbd1817.out0 disk.drbd1931.out0 disk.sda.in0 disk.drbd1285.in0 disk.dm-0.in 2.47265625 disk.drbd1828.in02A2- _wandbruntime2915  _runtime2915t+ ' lr-AdamW/pg10.00016878204929226962 ' lr-AdamW/pg20.00016878204929226962 trainer/global_step6549 $ _timestamp1.785798202290402e+09 _runtime2915.113358599 _step520]3 ' lr-AdamW/pg10.00016878204929226962 ' lr-AdamW/pg20.00016878204929226962 trainer/global_step6549 $ _timestamp1.785798202290402e+09 _runtime2915.113358599 _step5202- _wandbruntime2915  _runtime2915"'O F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed118209913435648 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5748985595968 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed112446823343104 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated112712464278528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5918939192832 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5763177522176 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5763090092544 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed118209913435648 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 _runtime 2915.11469493 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed69164911 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed18756245 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 5763090092544 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed112712439374848 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 5749071399092 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated112446840382464 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated109652690412832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 % _timestamp1.7857982022913942e+09 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed69164911 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated25789747 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5763177522176 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 112681138 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed25789745 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated118631536242688 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated118210017904640 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 112690796 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed109652673373472 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated69174567 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated43516229 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 5919071964160 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed115401658969440 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current241 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current239(wO 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 112690796 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed109652673373472 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated69174567 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated43516229 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 5919071964160 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed115401658969440 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current241 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current239 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed118209913435648 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5748985595968 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed112446823343104 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated112712464278528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5918939192832 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5763177522176 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5763090092544 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed118209913435648 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 _runtime 2915.11469493 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed69164911 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed18756245 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 5763090092544 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed112712439374848 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 5749071399092 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated112446840382464 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated109652690412832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 % _timestamp1.7857982022913942e+09 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed69164911 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated25789747 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5763177522176 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 112681138 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed25789745 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated118631536242688 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated118210017904640g2- _wandbruntime2915  _runtime2915l&M A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5763969712128 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated118228064294400 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed69175434 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5919970198016 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated18759362 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 5763969712128 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current261 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated112464007145472 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated69185104 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated118649641593344 _step522 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5749948872426 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5764057148928 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 5749863069190 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current263 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 5919843725312 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 112707976 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated43522872 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated25793684 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 126472704 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed43522870 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated112464007145472 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5764057148928 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 151376384 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 trainer/global_step6549 % _timestamp1.7857982026612246e+09 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated118228064294400 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed112463990106112 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 112707976 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 112698304 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed69175434 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed44552783 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed109669413545792 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated44553046 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0&M H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated44553046 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5763969712128 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated118228064294400 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed69175434 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5919970198016 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated18759362 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 5763969712128 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current261 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated112464007145472 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated69185104 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated118649641593344 _step522 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5749948872426 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5764057148928 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 5749863069190 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current263 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 5919843725312 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 112707976 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated43522872 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated25793684 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 126472704 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed43522870 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated112464007145472 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5764057148928 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 151376384 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 trainer/global_step6549 % _timestamp1.7857982026612246e+09 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated118228064294400 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed112463990106112 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 112707976 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 112698304 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed69175434 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed44552783 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed109669413545792X[o2- _wandbruntime2915  _runtime2915vm _step523 &loss/total_loss21.117983112106238 epoch0 0loss/FRt1/assignment_loss3.5367485258314346 1loss/SRqqt1/assignment_loss1.124986255913973 2loss/SRqqt2/assignment_loss1.3379188448014343 3loss/FBt1/assignment_loss0.0065804299207793075 /loss/FRt1/detection_loss2.5376296712292565 1loss/FBt2/assignment_loss0.20038238447062895 /loss/FRt2/detection_loss2.4591010455642976 /loss/FBt1/detection_loss1.0784566621170253 /loss/FBt2/detection_loss2.8667805045843124 trainer/global_step6549 /loss/FRt2/assignment_loss2.382385480788446 1loss/SRqqt1/detection_loss2.4752296388164394 /loss/SRqqt2/detection_loss1.11178366806821 % _timestamp1.7857982026628506e+09 _runtime2915.485687319G6  1loss/SRqqt1/assignment_loss1.124986255913973 2loss/SRqqt2/assignment_loss1.3379188448014343 3loss/FBt1/assignment_loss0.0065804299207793075 /loss/FRt1/detection_loss2.5376296712292565 1loss/FBt2/assignment_loss0.20038238447062895 /loss/FRt2/detection_loss2.4591010455642976 /loss/FBt1/detection_loss1.0784566621170253 /loss/FBt2/detection_loss2.8667805045843124 trainer/global_step6549 /loss/FRt2/assignment_loss2.382385480788446 1loss/SRqqt1/detection_loss2.4752296388164394 /loss/SRqqt2/detection_loss1.11178366806821 % _timestamp1.7857982026628506e+09 _runtime2915.485687319 _step523 &loss/total_loss21.117983112106238 epoch0 0loss/FRt1/assignment_loss3.5367485258314346TDA(: ļU gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts96.254' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent32.084666666666664 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0VT(:  ļU network.recv398567 disk.drbd1552.in0 disk.drbd1550.in0) disk./.usagePercent32.34385073812845 disk.drbd1817.out0 disk.rbd1.out 11.6083984375 disk.sda1.in0 disk.nvme0n1.in 2.47265625 disk.drbd1828.in0 disk.sda.in0$ disk./.usageGB578.3409004211426 disk.drbd1510.out0 disk.drbd1584.out0 disk.rbd1.in0 disk.sda.out 79.20703125 disk.drbd1510.in0 disk.drbd1226.out0" disk.nvme0n1.out 136.685546875 disk.sda2.out 79.20703125 network.sent 5.994027e+06 disk.dm-0.out 136.685546875 disk.drbd1226.in0 disk.drbd2078.in0 disk.drbd2111.in0 disk.drbd1965.in0 disk.drbd1931.in0 disk.drbd1931.out0 disk.drbd2118.in0 disk.drbd1817.in0 disk.drbd1673.in0 disk.drbd1552.out0 disk.drbd1550.out0) proc.memory.availableMB 38512.2265625 cpu13.881192448090163 disk.drbd1938.out0 disk.rbd0.out0 disk.sda1.out0 disk.drbd1673.out0 disk.drbd1356.in0 disk.drbd1356.out0 disk.drbd2041.out0 disk.drbd1828.out0! proc.memory.rssMB 11260.40625* proc.memory.percent17.182016372680664 disk.drbd2041.in0 disk.rbd0.in0# memory_percent41.2350058555603 disk.drbd1285.in0 disk.drbd1285.out0 disk.drbd2118.out0 disk.drbd2111.out0 disk.drbd1289.out0 disk.drbd1938.in0 disk.dm-0.in 2.47265625 disk.drbd1584.in0 disk.drbd1965.out0 disk.drbd1289.in0 disk.sda2.in0 proc.cpu.threads29 disk.drbd2078.out0to2- _wandbruntime2937  _runtime2937Ӱ ' lr-AdamW/pg10.00017007065861653493 ' lr-AdamW/pg20.00017007065861653493 trainer/global_step6599 $ _timestamp1.785798224555778e+09 _runtime2937.378825999 _step524A< ' lr-AdamW/pg20.00017007065861653493 trainer/global_step6599 $ _timestamp1.785798224555778e+09 _runtime2937.378825999 _step524 ' lr-AdamW/pg10.00017007065861653493[A2- _wandbruntime2937  _runtime2937eM%O QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated44887452 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed113305161493504 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed119112247721472 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5807173657600 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed113572795214848 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated113305178532864 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated18900855 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed119536890646016 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed110489681989472 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed69692913 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated116282658891432 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 5964236591104 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed69692913 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated69702569 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed113305161493504 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated119112352190464 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current260 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated113572820118528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5964095431168 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current262 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed25986595 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 113550948 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated110489699028832 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated43848379 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed43848377 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated43848379 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 trainer/global_step6599 _step525 G>De viceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated69702569 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed119112247721472 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed44887190 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5792874059476 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 113541290 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5807086227968 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed18900595 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated119112352190464 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 % _timestamp1.7857982245568473e+09 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 141159936 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated113305178532864 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5807173657600 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed43848377 ^p^'O L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed69692913 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated69702569 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed113305161493504 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated119112352190464 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current260 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated113572820118528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 5964095431168 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current262 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed25986595 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 113550948 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated110489699028832 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated43848379 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed43848377 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated43848379 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 trainer/global_step6599 _step525 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated69702569 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed119112247721472 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed44887190 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5792874059476 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 113541290 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5807086227968 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed18900595 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated119112352190464 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 % _timestamp1.7857982245568473e+09 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 141159936 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated113305178532864 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5807173657600 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed43848377 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated44887452 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed113305161493504 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed119112247721472 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5807173657600 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed113572795214848 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated113305178532864 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated18900855 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed119536890646016 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed110489681989472 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed69692913 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated116282658891432 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 5964236591104 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 I2- _wandbruntime2937  _runtime2937}&M NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated69713106 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5807965847552 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current235 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated113590027235328 _runtime2937.752013637 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 113568128 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 113558456 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current233 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5965130096128 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 5964995234816 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 113558456 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed43855020 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated119555157331456 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated113322345295872 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed113322328256512 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed43855020 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed44893926 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed110506422161792 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5808053284352 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated18903627 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated69713106 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 trainer/global_step6599 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5808053284352 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated110506439201152 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5793837335934 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed119130294104064 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0qäK!M M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 trainer/global_step6599 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5808053284352 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated110506439201152 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5793837335934 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed119130294104064 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated69713106 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5807965847552 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current235 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated113590027235328 _runtime2937.752013637 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 113568128 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 113558456 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current233 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 5965130096128 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 5964995234816 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 113558456 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed43855020 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated119555157331456 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated113322345295872 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed113322328256512 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed43855020 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed44893926 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed110506422161792 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5808053284352 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated18903627 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated69713106 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 EDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed120014582007296 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed117163453128456 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed70220915 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed114163499643904 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated114433175958528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated44180529 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 5836848326108 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 trainer/global_step6649 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5851169793024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current253 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed120442472989696 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 6009452609024 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed114163499643904 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed120014582007296 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated70230571 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed26183445 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 % _timestamp1.7857982463015044e+09 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated114163516683264 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 114411100 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed44180527 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 6009321934848 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated111326707644832 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed44180527 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5851169793024 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed70220915 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated120442628567552 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed111326690605472 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 114401442 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated114163516683264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5836762522984 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated45227032 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated70230571 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated19043585 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current251 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5851082363392 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated26183447 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 _runtime2959.127670311 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed114433151054848 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated120014686476288 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated120014686476288 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed45226779 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed120014582007296 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed117163453128456 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed70220915 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed114163499643904 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated114433175958528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated44180529 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 5836848326108 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 trainer/global_step6649 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5851169793024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current253 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed120442472989696 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 6009452609024 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed114163499643904 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed120014582007296 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated70230571 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed26183445 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 % _timestamp1.7857982463015044e+09 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated114163516683264 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 114411100 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed44180527 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 6009321934848 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated111326707644832 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed44180527 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5851169793024 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed70220915 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated120442628567552 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed111326690605472 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 114401442 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated114163516683264 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5836762522984 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated45227032 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18P<2- _wandbruntime2959  _runtime2959SM WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated120032732866048 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated19046436 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated120460741567488 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed114180666406912 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5852049419776 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 5837639996206 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated120032732866048 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 6010358492160 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated70241108 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated26187384 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated117181173616594 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed120460581802496 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 % _timestamp1.7857982466833694e+09 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed26187382 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed45233549 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5852049419776 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated111343447817152 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 _runtime2959.506761419 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 114428280 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed114180666406912 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated114180683446272 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 114418608 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated45233820 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 trainer/global_step6649 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5851961982976 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 5851961982976 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed114450358171648 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 114418608 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated70241108 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed120032628389888 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current271 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed111343430777792 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed70231438 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current269 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5837725799442 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 114428280 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 114428280 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed114180666406912 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated114180683446272 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 114418608 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated45233820 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 trainer/global_step6649 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5851961982976 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 5851961982976 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed114450358171648 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 114418608 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated70241108 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed120032628389888 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current271 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed111343430777792 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed70231438 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current269 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5837725799442 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 114428280 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed114180666406912 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5852049419776 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 5837639996206 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated120032732866048 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 6010358492160 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated70241108 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated26187384 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated117181173616594 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed120460581802496 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 % _timestamp1.7857982466833694e+09 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed26187382 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed45233549 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5852049419776 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated111343447817152 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 _runtime2959.506761419 EDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed26380295 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated44512679 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated115293531798528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5880650986492 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed115021837794304 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated112163716260832 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 5880736789616 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed120916916293120 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed115293506894848 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 _runtime2980.909772502 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current268 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated19187064 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed115021837794304 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed19186796 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 6054537059840 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated44512679 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed45567091 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 trainer/global_step6699 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 6054671928320 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 115271252 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated118044453050448 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5895078498816 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated120917020762112 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 5895078498816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed26380295 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated44512679 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated115293531798528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5880650986492 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed115021837794304 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated112163716260832 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 5880736789616 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed120916916293120 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed115293506894848 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 _runtime2980.909772502 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current268 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated19187064 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed115021837794304 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed19186796 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 6054537059840 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated44512679 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed45567091 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 trainer/global_step6699 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 6054671928320 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 115271252 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated118044453050448 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5895078498816 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated120917020762112 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 5895078498816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 J6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated19189969 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated70769110 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed26384232 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed45573946 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed115310714011648 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed120934962675712 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 115288432 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 115278760 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 trainer/global_step6699 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed70759440 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated112180456433152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated120935067151872 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated120935067151872 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated45574203 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated121366310535168 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed121366161255936 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 6055447244288 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current255 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 5881528459714 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed118061967853506 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated115039021596672 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 124375552 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated118062070696102 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed112180439393792 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 % _timestamp1.7857982684607272e+09|B&M J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current257 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed70759440 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated19189969 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated70769110 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed26384232 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed45573946 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed115310714011648 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed120934962675712 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 115288432 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 115278760 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 trainer/global_step6699 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed70759440 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated112180456433152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated120935067151872 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated120935067151872 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated45574203 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated121366310535168 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed121366161255936 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 6055447244288 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current255 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 5881528459714 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed118061967853506 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated115039021596672 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 124375552 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated118062070696102 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed112180439393792 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 % _timestamp1.7857982684607272e+09 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated44519322 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed115039004557312 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 115278760 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 149279232 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated26384234 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated115310738915328 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0|T2- _wandbruntime2981  _runtime2981 &loss/total_loss19.333610265092723 epoch0 /loss/FBt1/assignment_loss0.052407470788903 /loss/FRt2/detection_loss2.5753875174629153 1loss/SRqqt2/detection_loss0.9952623946650058 /loss/FBt1/detection_loss1.0495006369559892 /loss/FBt2/detection_loss0.3389227390296043 /loss/FRt1/assignment_loss3.682940803294958 _step535 /loss/FRt2/assignment_loss2.834651156147914 2loss/SRqqt2/assignment_loss1.2228794885070429 0loss/FBt2/assignment_loss0.2917527476325631 trainer/global_step6699 $ _timestamp1.785798268462365e+09 _runtime 2981.28519385 2loss/SRqqt1/assignment_loss1.6001720190048219 /loss/FRt1/detection_loss2.6355640884748723 1loss/SRqqt1/detection_loss2.0541692031281333Dz? /loss/FBt1/detection_loss1.0495006369559892 /loss/FBt2/detection_loss0.3389227390296043 /loss/FRt1/assignment_loss3.682940803294958 _step535 /loss/FRt2/assignment_loss2.834651156147914 2loss/SRqqt2/assignment_loss1.2228794885070429 0loss/FBt2/assignment_loss0.2917527476325631 trainer/global_step6699 $ _timestamp1.785798268462365e+09 _runtime 2981.28519385 2loss/SRqqt1/assignment_loss1.6001720190048219 /loss/FRt1/detection_loss2.6355640884748723 1loss/SRqqt1/detection_loss2.0541692031281333 &loss/total_loss19.333610265092723 epoch0 /loss/FBt1/assignment_loss0.052407470788903 /loss/FRt2/detection_loss2.5753875174629153 1loss/SRqqt2/detection_loss0.9952623946650058)V(:  ǭU disk.drbd2118.in0 network.sent 6.132058e+06 disk.drbd1938.out0$ disk./.usageGB578.3438453674316 disk.drbd2078.in0$ disk.nvme0n1.out142.00048828125 proc.cpu.threads29 network.recv406414 disk.sda.out 80.80859375 disk.drbd1285.out0 disk.drbd1356.in0! proc.memory.rssMB 11260.40625 disk.rbd1.out 11.796875) disk./.usagePercent32.34401543494159 disk.drbd1931.in0 disk.sda1.out0 disk.drbd1965.in0 disk.drbd1931.out0 disk.drbd1828.out0 disk.rbd0.out0 disk.nvme0n1.in 2.47265625 disk.drbd1828.in0 disk.drbd1550.in0 disk.drbd1550.out0 disk.drbd1289.out0 disk.drbd1938.in0 disk.sda.in0 disk.drbd2078.out0 disk.rbd0.in0 disk.drbd1584.in0 disk.sda2.out 80.80859375* proc.memory.availableMB38512.00390625 disk.drbd1285.in0 disk.drbd1552.out0 disk.drbd2111.out0 disk.drbd1965.out0 cpu13.875416443786207 disk.sda1.in0 disk.dm-0.in 2.47265625 disk.drbd1673.out0 disk.drbd2041.in0 disk.drbd2041.out0 disk.drbd1584.out0 disk.drbd2118.out0 disk.sda2.in0 disk.rbd1.in0 disk.drbd1552.in0 disk.drbd1289.in0 disk.drbd1817.out0 disk.drbd2111.in0 disk.drbd1817.in0 disk.drbd1226.out0* proc.memory.percent17.182016372680664 disk.drbd1673.in0 disk.drbd1510.out0 disk.drbd1226.in0 disk.drbd1356.out0$ memory_percent41.23534560203552 disk.drbd1510.in0! disk.dm-0.out142.00048828125(fW(: V gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts105.858' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent35.286 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0(R(:  ǁV disk.drbd1510.out0 disk.rbd1.in0 disk.drbd2111.in0 disk.drbd1673.out0 disk.drbd1550.out0 disk.drbd1938.in0 disk.drbd2078.out0 disk.drbd1356.out0 disk.drbd1965.in0 disk.drbd1965.out0$ disk./.usageGB578.3440704345703 disk.drbd1289.out0 disk.sda.in0 disk.sda1.in0 disk.drbd2118.out0 disk.rbd1.out 11.890625! proc.memory.rssMB 11260.40625 disk.drbd2041.in0 disk.sda.out 81.5234375 disk.dm-0.in 2.47265625 disk.sda2.out 81.5234375 disk.drbd1550.in0 disk.drbd1285.in0 disk.drbd1285.out0 proc.cpu.threads29 disk.drbd1931.in0 disk.drbd1226.out0 disk.drbd2118.in0# disk.nvme0n1.out142.8017578125 disk.drbd2111.out0* disk./.usagePercent32.344028021874195 disk.drbd1552.in0 disk.drbd1584.in0 disk.dm-0.out142.8017578125 disk.drbd1938.out0 disk.drbd2078.in0 disk.nvme0n1.in 2.47265625 disk.drbd1356.in0 disk.drbd1817.in0 disk.drbd1828.in0 disk.rbd0.in0 disk.drbd1584.out0 disk.sda2.in0) proc.memory.availableMB 38511.5078125* proc.memory.percent17.182016372680664 disk.sda1.out0 disk.drbd2041.out0 disk.drbd1828.out0 disk.drbd1289.in0 disk.drbd1510.in0 disk.drbd1817.out0 network.recv408863 cpu13.873904423119273 disk.drbd1552.out0$ memory_percent41.23610258102417 network.sent 6.177139e+06 disk.rbd0.out0 disk.drbd1226.in0 disk.drbd1673.in0 disk.drbd1931.out0.(: ûU gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts121.063' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent40.35433333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0. ُuncorrectedMemoryErrors0Kٲ2- _wandbruntime3003  _runtime3003bV trainer/global_step6749 $ _timestamp1.785798290236773e+09 _runtime3003.059743252 _step536 ' lr-AdamW/pg10.00017393648658933086 ' lr-AdamW/pg20.00017393648658933086 $ _timestamp1.785798290236773e+09 _runtime3003.059743252 _step536 ' lr-AdamW/pg10.00017393648658933086 ' lr-AdamW/pg20.00017393648658933086 trainer/global_step6749krm2- _wandbruntime3003  _runtime30036H80'O [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 5939162063872 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 _runtime3003.060993566 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5939162063872 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated116153887638528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated71286575 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated121819355047936 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5939074634240 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated44844829 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed121819250578944 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated115880192984064 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed71276919 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 6099716637184 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated113000724876832 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 % _timestamp1.7857982902377062e+09 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current266 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5924539450000 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 _step537 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated44844829 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed118925247287472 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 116121746 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated122253730755584 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated45907169 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 _runtime3003.060993566 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5939162063872 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated116153887638528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated71286575 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated121819355047936 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5939074634240 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated44844829 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed121819250578944 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated115880192984064 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed71276919 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 6099716637184 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated113000724876832 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 % _timestamp1.7857982902377062e+09 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current266 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5924539450000 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 _step537 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated44844829 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed118925247287472 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 116121746 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated122253730755584 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated45907169 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated118942967775610 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed26581082 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 5939954253824 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current252 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5925502726458 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated115897359747072 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5940041690624 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed122271683671040 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed71287442 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5939954253824 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed115897342707712 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed44851470 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current250 trainer/global_step6749 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed121837296961536 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5940041690624 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 116138912 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated121837401437696 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated116171094755328 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 6100613819392 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 5925416923222 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated44851472 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated113017465049152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 % _timestamp1.7857982906099765e+09 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed19332591 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed45913673 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed115897342707712 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated115897359747072 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated122271843436032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 116148584 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated26581084 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed116171069851648 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed71287442 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed113017448009792 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated71297112 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed121837296961536 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5940041690624 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 116138912 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated121837401437696 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated116171094755328 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 6100613819392 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 5925416923222 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocXQQated44851472 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated113017465049152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 % _timestamp1.7857982906099765e+09 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed19332591 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed45913673 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed115897342707712 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated115897359747072 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated122271843436032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 116148584 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated26581084 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed116171069851648 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed71287442 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed113017448009792 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated71297112 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated118942967775610 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed26581082 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 5939954253824 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current252 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5925502726458 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated115897359747072 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5940041690624 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed122271683671040 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed71287442 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816!Y2- _wandbruntime3003  _runtime3003 s9M 1loss/FBt1/assignment_loss0.06691043030162272 .loss/FRt2/detection_loss2.508377379913261 trainer/global_step6749 _runtime3003.434451309 _step539 1loss/FBt2/assignment_loss0.05338247436923874 1loss/SRqqt2/detection_loss1.1015338059859696 % _timestamp1.7857982906116114e+09 /loss/FRt2/assignment_loss2.690074062865713 1loss/SRqqt2/assignment_loss1.426311204669836 1loss/SRqqt1/detection_loss1.6809731893369864 /loss/FBt1/detection_loss0.9477518477201969 epoch0 /loss/FRt1/assignment_loss3.847993761022514 1loss/SRqqt1/assignment_loss1.566346302114684 /loss/FRt1/detection_loss2.4396448717496106 /loss/FBt2/detection_loss1.8574655414558947 %loss/total_loss20.18676487150553&z1 _step539 1loss/FBt2/assignment_loss0.05338247436923874 1loss/SRqqt2/detection_loss1.1015338059859696 % _timestamp1.7857982906116114e+09 /loss/FRt2/assignment_loss2.690074062865713 1loss/SRqqt2/assignment_loss1.426311204669836 1loss/SRqqt1/detection_loss1.6809731893369864 /loss/FBt1/detection_loss0.9477518477201969 epoch0 /loss/FRt1/assignment_loss3.847993761022514 1loss/SRqqt1/assignment_loss1.566346302114684 /loss/FRt1/detection_loss2.4396448717496106 /loss/FBt2/detection_loss1.8574655414558947 %loss/total_loss20.18676487150553 1loss/FBt1/assignment_loss0.06691043030162272 .loss/FRt2/detection_loss2.508377379913261 trainer/global_step6749 _runtime3003.434451309߳(: U gpu.0.gpu34.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts97.303' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent32.434333333333335 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0 S(:  U disk.drbd1510.in0 disk.drbd2041.out0* proc.memory.percent17.182016372680664) disk./.usagePercent32.34405383575294 disk.sda.in0 disk.drbd2078.in0 disk.drbd1828.in0 network.sent 6.222162e+06 cpu13.872573943823674 network.recv411346 disk.drbd1285.out0 disk.nvme0n1.in 2.47265625 disk.sda2.in0 disk.drbd1965.out0 disk.drbd1931.out0 disk.sda.out 81.8203125 disk.drbd1938.in0 disk.rbd0.in0 disk.sda2.out 81.8203125 disk.drbd1965.in0 disk.drbd1673.out0 disk.drbd1931.in0! proc.memory.rssMB 11260.40625 disk.drbd1289.in0 disk.drbd1938.out0 disk.sda1.out0 disk.drbd1285.in0 disk.drbd1226.in0 disk.drbd1584.in0 disk.drbd2118.out0 disk.drbd1226.out0 disk.rbd1.out 12.0390625 disk.drbd1828.out0 disk.drbd1289.out0 disk.dm-0.out143.1455078125 disk.drbd1584.out0 disk.drbd2118.in0 disk.drbd1817.in0 disk.drbd1817.out0 disk.drbd1552.in0 disk.rbd0.out0# disk.nvme0n1.out143.1455078125 disk.drbd1356.in0 disk.drbd2111.in0 disk.drbd1552.out0$ memory_percent41.23621582984924* proc.memory.availableMB38511.43359375 disk.sda1.in0 disk.dm-0.in 2.47265625 disk.drbd1673.in0$ disk./.usageGB578.3445320129395 disk.drbd1356.out0 disk.rbd1.in0 disk.drbd1550.in0 disk.drbd1550.out0 disk.drbd2041.in0 proc.cpu.threads29 disk.drbd1510.out0 disk.drbd2078.out0 disk.drbd2111.out0r!/2- _wandbruntime3024  _runtime3024  trainer/global_step6799 $ _timestamp1.785798311766784e+09 _runtime3024.589751391 _step540 ' lr-AdamW/pg10.00017522509591359615 ' lr-AdamW/pg20.00017522509591359615?v ' lr-AdamW/pg10.00017522509591359615 ' lr-AdamW/pg20.00017522509591359615 trainer/global_step6799 $ _timestamp1.785798311766784e+09 _runtime3024.589751391 _step540ej2- _wandbruntime3024  _runtime3024w&'O F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 116981898 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5968427913508 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed45176977 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated113837733492832 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 _runtime3024.591021893 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated71814577 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 147189248 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current266 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated119806247209464 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed122721584864768 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed46247270 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5983070769664 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated117014243478528 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed122721584864768 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 5983070769664 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 5968513716632 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 116991556 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed116738514095104 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated45176979 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed45176977 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5983158199296 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated116738531134464 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated26773997 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated19473539 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed71804921 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated122721689333760 % _timestamp1.7857983117677255e+09 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0f'O C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed113837716453472 trainer/global_step6799 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 5968513716632 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 116991556 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed116738514095104 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated45176979 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed45176977 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 5983158199296 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated116738531134464 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated26773997 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated19473539 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed71804921 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated122721689333760 % _timestamp1.7857983117677255e+09 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 116981898 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 5968427913508 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed45176977 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated113837733492832 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 _runtime3024.591021893 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated71814577 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 147189248 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current266 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated119806247209464 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed122721584864768 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed46247270 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 5983070769664 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated117014243478528 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed122721584864768 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 5983070769664 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed122739631247360 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated122739735723520 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5984037826048 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated19476425 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 117008736 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated113854473665152 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5984037826048 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated123177383899648 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed45183620 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed116755680858112 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed46254101 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated116755697897472 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 116999064 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated117031450595328 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed19476169 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed71815444 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 _step542 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 5969305386730 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 trainer/global_step6799 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5983950389248 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current258 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed123177230426112 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed113854456625792 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856o&M M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed122739631247360 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated122739735723520 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 5984037826048 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated19476425 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 117008736 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated113854473665152 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 5984037826048 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated123177383899648 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed45183620 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed116755680858112 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed46254101 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated116755697897472 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 116999064 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated117031450595328 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed19476169 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed71815444 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 _step542 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 5969305386730 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 trainer/global_step6799 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 5983950389248 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current258 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed123177230426112 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed113854456625792 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed26777932 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 % _timestamp1.7857983121392572e+09 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 117008736 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated122739735723520 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed119823762012522 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 5983950389248 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed116755680858112 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current256 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 6145933304320 _runtime3024.962628701 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated71825114 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated116755697897472 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated71825114 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated26777934 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed117031425691648 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 5969391189966 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated46254359 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated119823864855118 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed45183620 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392̻2- _wandbruntime3024  _runtime3024 ԡ /loss/FBt2/detection_loss1.5935832001268864 % _timestamp1.7857983121409123e+09 0loss/FRt2/assignment_loss2.5846639901776856 1loss/SRqqt2/assignment_loss1.167624597035108 1loss/FBt1/assignment_loss0.08437173724463953 /loss/FBt1/detection_loss1.3390214452811051 epoch0 2loss/SRqqt1/assignment_loss1.7200211758414905 -loss/FRt2/detection_loss2.32245236050479 trainer/global_step6799 _runtime3024.963757014 _step543 0loss/FRt1/assignment_loss3.6795811700348806 /loss/FRt1/detection_loss2.4349460017563094 %loss/total_loss20.32026675231915 1loss/FBt2/assignment_loss0.24954331305343658 1loss/SRqqt1/detection_loss2.1018445836725057 1loss/SRqqt2/detection_loss1.0426131775903125%J  _step543 0loss/FRt1/assignment_loss3.6795811700348806 /loss/FRt1/detection_loss2.4349460017563094 %loss/total_loss20.32026675231915 1loss/FBt2/assignment_loss0.24954331305343658 1loss/SRqqt1/detection_loss2.1018445836725057 1loss/SRqqt2/detection_loss1.0426131775903125 /loss/FBt2/detection_loss1.5935832001268864 % _timestamp1.7857983121409123e+09 0loss/FRt2/assignment_loss2.5846639901776856 1loss/SRqqt2/assignment_loss1.167624597035108 1loss/FBt1/assignment_loss0.08437173724463953 /loss/FBt1/detection_loss1.3390214452811051 epoch0 2loss/SRqqt1/assignment_loss1.7200211758414905 -loss/FRt2/detection_loss2.32245236050479 trainer/global_step6799 _runtime3024.963757014oP(: ֠U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts123.834' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent41.278000000000006 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0RO(:  U! proc.memory.rssMB 11260.40625 disk.sda2.out 82.390625 disk.drbd1673.out0 disk.drbd1828.in0 disk.drbd1289.in0 disk.drbd1285.out0 disk.drbd1356.in0 disk.drbd1673.in0 disk.drbd1938.out0 disk.sda1.out0 disk.drbd1226.in0 network.sent 6.225283e+06 disk.drbd1584.out0 disk.rbd0.in0 disk.drbd1285.in0 disk.drbd1510.in0 disk.drbd2118.in0 disk.drbd1356.out0 disk.drbd2041.in0 disk.sda.in0 disk.sda.out 82.390625 network.recv412460 disk.drbd1584.in0 disk.drbd1817.in0 disk.drbd1828.out0 disk.rbd0.out0 disk.drbd2078.in0$ disk.nvme0n1.out144.06982421875 disk.drbd1817.out0 disk.drbd1931.in0$ disk./.usageGB578.3445930480957 disk.rbd1.out12.125 disk.drbd1550.in0! disk.dm-0.out144.06982421875 disk.drbd1552.out0 disk.sda1.in0 cpu13.871050575449665 disk.drbd2118.out0 disk.sda2.in0 disk.drbd1965.in0 disk.dm-0.in 2.47265625* proc.memory.percent17.182016372680664 disk.drbd1965.out0 disk.drbd1226.out0 disk.nvme0n1.in 2.47265625 disk.drbd1289.out0$ memory_percent41.23626351356506 disk.drbd2078.out0 disk.rbd1.in0 disk.drbd1552.in0 disk.drbd1938.in0* proc.memory.availableMB38511.40234375 disk.drbd2111.in0 disk.drbd1550.out0) disk./.usagePercent32.34405724915839 proc.cpu.threads29 disk.drbd2111.out0 disk.drbd1931.out0 disk.drbd2041.out0 disk.drbd1510.out0o(: ڸU gpu.0.gpu36.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts112.449' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent37.483 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors02pO(:  U' proc.memory.availableMB 38511.40625$ disk./.usageGB578.3445358276367 disk.drbd2111.out0 disk.rbd0.in0 disk.drbd2118.out0 disk.drbd1817.in0 disk.drbd1965.out0 disk.drbd1552.in0 disk.nvme0n1.in 2.47265625 network.sent 6.27037e+06 disk.sda1.in0 disk.drbd2118.in0 disk.sda.in0 disk.drbd1285.in0 disk.drbd1510.in0 disk.drbd1356.in0 disk.sda2.out 82.8359375 disk.drbd1285.out0 proc.cpu.threads29 disk.drbd1828.out0 disk.sda.out 82.8359375 disk.drbd1584.out0 disk.drbd1938.in0 disk.rbd1.out12.1875 disk.drbd1552.out0 disk.drbd1938.out0 disk.drbd2078.in0 disk.sda2.in0 disk.dm-0.in 2.47265625 cpu13.869582808721926 disk.drbd1817.out0 disk.drbd1673.in0% memory_percent41.236257553100586 disk.drbd1931.in0 disk.drbd2041.out0 disk.drbd1510.out0 disk.drbd1356.out0 disk.drbd2111.in0 disk.drbd1828.in0 disk.drbd1289.in0 disk.sda1.out0 disk.drbd1226.in0 disk.drbd2078.out0! proc.memory.rssMB 11260.40625 disk.drbd1289.out0) disk./.usagePercent32.34405404909077 disk.rbd1.in0 disk.drbd2041.in0 disk.rbd0.out0* proc.memory.percent17.182016372680664 disk.drbd1931.out0 disk.drbd1550.in0 disk.drbd1550.out0! disk.dm-0.out144.21826171875 disk.drbd1584.in0$ disk.nvme0n1.out144.21826171875 network.recv414828 disk.drbd1965.in0 disk.drbd1673.out0 disk.drbd1226.out0v2- _wandbruntime3046  _runtime30464\v ' lr-AdamW/pg20.00017651370523786146 trainer/global_step6849 % _timestamp1.7857983339520366e+09 _runtime3046.775001931 _step544 ' lr-AdamW/pg10.00017651370523786146V-o ' lr-AdamW/pg20.00017651370523786146 trainer/global_step6849 % _timestamp1.7857983339520366e+09 _runtime3046.775001931 _step544 ' lr-AdamW/pg10.00017651370523786146v2- _wandbruntime3046  _runtime3046p*eO N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 117851708 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed117874574414848 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated117874599318528 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed120687041446488 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated72342579 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6012402180140 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed46587944 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current234 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed72332923 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6027154334720 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed72332923 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated114674742108832 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed117596852245504 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current236 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed19617099 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.s%ϽEmall_pool.freed0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated123624023619584 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 6190085666304 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated26970847 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed114674725069472 trainer/global_step6849 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed117596852245504 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 6027066905088 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6027066905088 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 % _timestamp1.7857983339529488e+09 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 6012316377016 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed45509127 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed123623919150592 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed26970845 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated19617333 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed124064660081152 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated45509129 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 6190220534784 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 _step545 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated123624023619584 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated123624023619584 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 6190085666304 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated26970847 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed114674725069472 trainer/global_step6849 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed117596852245504 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 6027066905088 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6027066905088 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 % _timestamp1.7857983339529488e+09 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 6012316377016 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed45509127 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed123623919150592 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed26970845 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated19617333 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed124064660081152 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated45509129 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 6190220534784 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 _step545 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated123624023619584 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated117874599318528 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed120687041446488 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated72342579 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6012402180140 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed46587944 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current234 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed72332923 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6027154334720 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed72332923 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated114674742108832 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed117596852245504 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current236 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed19617099 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680Z~2- _wandbruntime3047  _runtime3047D[&M OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed117614019008512 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6027946524672 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 117868888 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated26974784 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed26974782 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed120704659092030 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed45515770 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated117891806435328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 % _timestamp1.7857983343325202e+09 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated117614036047872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6027946524672 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 117859216 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed117891781531648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 _step546 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated19620211 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated114691482281152 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6013279653474 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6013193850238 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated124082927471104 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime 3047.1559173 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated123642070009344 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 6028033961472 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed123641965533184 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated120704761934626 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed117614019008512 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed45515770 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 117859216 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated72353116 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed46594734 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated72353116 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed72343446 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated46594995 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current259 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated117614036047872 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed114691465241792 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated45515772 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 6191121035776 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 117868888 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated45515772 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121Oik"M TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 117868888 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated45515772 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed117614019008512 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6027946524672 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 117868888 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated26974784 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed26974782 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed120704659092030 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed45515770 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated117891806435328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 % _timestamp1.7857983343325202e+09 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated117614036047872 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6027946524672 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 117859216 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed117891781531648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 _step546 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated19620211 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated114691482281152 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6013279653474 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6013193850238 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated124082927471104 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime 3047.1559173 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated123642070009344 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 6028033961472 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed123641965533184 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated120704761934626 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed117614019008512 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed45515770 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 117859216 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated72353116 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed46594734 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated72353116 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed72343446 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated46594995 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current259 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated117614036047872 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed114691465241792 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated45515772 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 6191121035776 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0Q2- _wandbruntime3047  _runtime3047L* _runtime3047.157208316 -loss/FRt1/detection_loss2.51391929594597 0loss/SRqqt1/detection_loss2.047920796771844 1loss/SRqqt2/detection_loss1.0254174052354168 /loss/FBt1/detection_loss1.0463251496905521 /loss/FBt2/detection_loss1.1002807815869653 0loss/FRt1/assignment_loss3.2103654765846708 2loss/SRqqt2/assignment_loss1.2633446758941693 /loss/FRt2/detection_loss2.5259642423810185 &loss/total_loss18.493638439661375 _step547 0loss/FRt2/assignment_loss2.0027562114032538 2loss/SRqqt1/assignment_loss1.3570708652337393 /loss/FBt2/assignment_loss0.299507372668207 trainer/global_step6849 1loss/FBt1/assignment_loss0.10076616626556936 epoch0 % _timestamp1.7857983343343618e+09Ƒ 1loss/SRqqt2/detection_loss1.0254174052354168 /loss/FBt1/detection_loss1.0463251496905521 /loss/FBt2/detection_loss1.1002807815869653 0loss/FRt1/assignment_loss3.2103654765846708 2loss/SRqqt2/assignment_loss1.2633446758941693 /loss/FRt2/detection_loss2.5259642423810185 &loss/total_loss18.493638439661375 _step547 0loss/FRt2/assignment_loss2.0027562114032538 2loss/SRqqt1/assignment_loss1.3570708652337393 /loss/FBt2/assignment_loss0.299507372668207 trainer/global_step6849 1loss/FBt1/assignment_loss0.10076616626556936 epoch0 % _timestamp1.7857983343343618e+09 _runtime3047.157208316 -loss/FRt1/detection_loss2.51391929594597 0loss/SRqqt1/detection_loss2.047920796771844K(: ˽U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts125.726' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent41.90866666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0sbS(:  ˽LjV disk.sda2.in0 disk.drbd2111.in0 disk.sda.out 83.49609375 disk.drbd2078.in0) disk./.usagePercent32.34405298240157 disk.drbd1673.out0 disk.rbd0.out0! disk.dm-0.out145.05712890625 disk.drbd1226.in0 network.recv417220 disk.drbd1965.in0 disk.drbd1965.out0 disk.drbd1673.in0$ disk.nvme0n1.out145.05712890625 disk.drbd2118.out0 disk.nvme0n1.in 2.47265625! proc.memory.rssMB 11260.40625 disk.drbd1931.in0 disk.drbd2041.in0 disk.drbd1552.in0 disk.drbd1828.out0 disk.rbd1.in0 disk.drbd1931.out0 disk.drbd1828.in0 disk.drbd1584.out0* proc.memory.percent17.182016372680664 disk.drbd1938.in0 disk.drbd1510.in0 disk.dm-0.in 2.47265625 disk.drbd2118.in0 disk.drbd1356.out0( proc.memory.availableMB 38510.859375 disk.drbd1550.in0 disk.drbd1289.out0 disk.sda1.out0 disk.drbd1356.in0 network.sent 6.315577e+06 disk.drbd1552.out0 disk.drbd1285.in0 disk.drbd1510.out0 disk.drbd2111.out0 disk.drbd1817.in0 disk.rbd1.out12.28125 disk.drbd1550.out0 disk.drbd1817.out0 disk.sda1.in0 disk.drbd1226.out0$ disk./.usageGB578.3445167541504 disk.sda2.out 83.49609375 disk.drbd2041.out0 disk.drbd1289.in0 disk.drbd1285.out0 disk.drbd2078.out0 proc.cpu.threads29 disk.drbd1584.in0$ memory_percent41.23709201812744 cpu13.868247888153524 disk.drbd1938.out0 disk.rbd0.in0 disk.sda.in092- _wandbruntime3068  _runtime3068op ' lr-AdamW/pg10.00017780231456212677 ' lr-AdamW/pg20.00017780231456212677 trainer/global_step6899 % _timestamp1.7857983560883715e+09 _runtime3068.911332412 _step548m _runtime3068.911332412 _step548 ' lr-AdamW/pg10.00017780231456212677 ' lr-AdamW/pg20.00017780231456212677 trainer/global_step6899 % _timestamp1.7857983560883715e+09ƯG2- _wandbruntime3068  _runtime3068k'O JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed118455190395904 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed46928827 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed115511733685472 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated121568041368480 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated45841279 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed118455190395904 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 118711860 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated118455207435264 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated46929087 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 _step549 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed45841277 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated72870581 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed118734930254848 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed124970167347712 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed72860925 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed124526253436416 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated118455207435264 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed27167695 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed72860925 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed19761132 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 6235367767040 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated115511750724832 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6056290643648 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed124526253436416 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 6071150470144 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 6235237092864 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated27167697 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated124526357905408 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6071150470144 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated19761390 % _timestamp1.7857983560893335e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 6071063040512 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated124526357905408 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 118702202 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated124970322925568 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated118734955158528 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed121567938525996 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current260 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated115511750724832 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6056290643648 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed124526253436416 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 6071150470144 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 6235237092864 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated27167697 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated124526357905408 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6071150470144 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated19761390 % _timestamp1.7857983560893335e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 6071063040512 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated124526357905408 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 118702202 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated124970322925568 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated118734955158528 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed121567938525996 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current260 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated121568041368480 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated45841279 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0bӾ2- _wandbruntime3069  _runtime30691 ,vM UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated45847922 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _step550 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed72871448 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated115528490897152 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed27171632 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed118752137371648 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6057168116982 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed118472357158912 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed46935658 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated124544404295168 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed118472357158912 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated46935923 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current263 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated124544404295168 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed124988276922368 trainer/global_step6899 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 118729040 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 118719368 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed45847920 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed121585556171538 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated118472374198272 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 118719368 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current265 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated121585659014134 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 6236139550720 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 XFDypf eviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6071942660096 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 118729040 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated45847922 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _step550 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed72871448 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated115528490897152 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed27171632 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed118752137371648 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6057168116982 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed118472357158912 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed46935658 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated124544404295168 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed118472357158912 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated46935923 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current263 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated124544404295168 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed124988276922368 trainer/global_step6899 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 118729040 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 118719368 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed45847920 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed121585556171538 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated118472374198272 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 118719368 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current265 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated121585659014134 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 6236139550720 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6071942660096 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 118729040 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated73398583 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed46173427 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated125428692191232 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6115146605568 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6115059175936 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated116348759340832 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed116348742301472 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed125428587722240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 _runtime3090.934842742 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed125428587722240 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated122448938447988up O R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed125428587722240 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated122448938447988 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated27364547 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed125875704103424 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated73398583 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed46173427 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated125428692191232 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6115146605568 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6115059175936 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated116348759340832 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed116348742301472 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed125428587722240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 _runtime3090.934842742~Œq2- _wandbruntime3091  _runtime3091".&M QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed27368482 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed46180070 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed19906643 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated119330712348672 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated47275374 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 119589192 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed125446634104832 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6115938795520 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 6281322596352 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed73399450 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6101056580490 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current249 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated73409120 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6116026232320 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated125893969281536 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 6281451166208 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _step554 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 119589192 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 119579520 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed46180070 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed119330695309312 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated122466556093642 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed47275125 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 % _timestamp1.7857983784811673e+09 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 6116026232320 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 trainer/global_step6949 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed122466453251046 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated46180072 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed119612493211648 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed73399450 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated119330712348672 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated73409120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392&M D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 119579520 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed46180070 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed119330695309312 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated122466556093642 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed47275125 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 % _timestamp1.7857983784811673e+09 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 6116026232320 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 trainer/global_step6949 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed122466453251046 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated46180072 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed119612493211648 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed73399450 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated119330712348672 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated73409120 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed27368482 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed46180070 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed19906643 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated119330712348672 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated47275374 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 119589192 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed125446634104832 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6115938795520 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 6281322596352 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed73399450 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6101056580490 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current249 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated73409120 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6116026232320 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated125893969281536 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 6281451166208 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _step554 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 119589192 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800['2- _wandbruntime3091  _runtime3091p;p 2loss/SRqqt1/assignment_loss0.9508264815020893 epoch0 _runtime3091.305784385 0loss/FRt2/assignment_loss2.3399182926667366 /loss/FRt1/detection_loss2.5870505539660757 /loss/FRt2/detection_loss2.5676943736248896 /loss/FBt2/detection_loss2.5256682435671487 &loss/total_loss20.645266047573745 trainer/global_step6949 0loss/FRt1/assignment_loss3.4245493931973234 1loss/SRqqt2/assignment_loss1.344742854577682 0loss/SRqqt1/detection_loss1.899374578064049 1loss/SRqqt2/detection_loss1.0649946579522283 /loss/FBt1/detection_loss1.0572058303514495 % _timestamp1.7857983784829493e+09 _step555 1loss/FBt1/assignment_loss0.01263617865436828 0loss/FBt2/assignment_loss0.8706046094497045'Ԁ  _runtime3091.305784385 0loss/FRt2/assignment_loss2.3399182926667366 /loss/FRt1/detection_loss2.5870505539660757 /loss/FRt2/detection_loss2.5676943736248896 /loss/FBt2/detection_loss2.5256682435671487 &loss/total_loss20.645266047573745 trainer/global_step6949 0loss/FRt1/assignment_loss3.4245493931973234 1loss/SRqqt2/assignment_loss1.344742854577682 0loss/SRqqt1/detection_loss1.899374578064049 1loss/SRqqt2/detection_loss1.0649946579522283 /loss/FBt1/detection_loss1.0572058303514495 % _timestamp1.7857983784829493e+09 _step555 1loss/FBt1/assignment_loss0.01263617865436828 0loss/FBt2/assignment_loss0.8706046094497045 2loss/SRqqt1/assignment_loss0.9508264815020893 epoch0(: U gpu.0.gpu34.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts128.59' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.86333333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0YYO(:  U disk.drbd2041.out0 cpu13.863419872087945 proc.cpu.threads29 disk.nvme0n1.in 2.47265625 disk.sda2.in0 disk.sda2.out84.40625 disk.drbd2111.out0* proc.memory.percent17.182016372680664* proc.memory.availableMB38511.16015625) disk./.usagePercent32.34413703751087 disk.drbd1938.out0 disk.drbd1356.in0# disk./.usageGB578.346019744873 disk.drbd1510.out0 disk.drbd1584.out0 disk.drbd2041.in0 disk.drbd1289.out0 disk.drbd1226.in0 disk.drbd1965.out0 disk.drbd1552.out0 disk.drbd1938.in0 disk.rbd0.in0 disk.sda1.in0 disk.dm-0.out 145.998046875 disk.drbd1828.out0 disk.drbd1510.in0 disk.drbd1226.out0 disk.drbd2078.out0 disk.sda1.out0 disk.drbd1817.in0! proc.memory.rssMB 11260.40625 disk.drbd1828.in0 disk.sda.out84.40625 disk.drbd1285.out0 disk.drbd1584.in0 disk.drbd1817.out0 disk.drbd1289.in0" disk.nvme0n1.out 145.998046875 disk.drbd1356.out0 disk.drbd1931.out0 disk.drbd1550.in0 disk.drbd2118.out0 disk.drbd1550.out0 disk.rbd0.out0 disk.rbd1.out 12.3759765625 disk.drbd1965.in0 disk.drbd1673.in0 disk.drbd1673.out0 disk.drbd1931.in0 disk.drbd2118.in0 network.sent 6.408913e+06 disk.dm-0.in 2.47265625 disk.drbd2078.in0 disk.rbd1.in0$ memory_percent41.23663306236267 network.recv422468 disk.sda.in0 disk.drbd1285.in0 disk.drbd1552.in0 disk.drbd2111.in0I5j2- _wandbruntime3113  _runtime3113* & lr-AdamW/pg20.0001803795332106574 trainer/global_step6999 % _timestamp1.7857984005278301e+09 _runtime3113.350806801 _step556 & lr-AdamW/pg10.0001803795332106574gA* & lr-AdamW/pg10.0001803795332106574 & lr-AdamW/pg20.0001803795332106574 trainer/global_step6999 % _timestamp1.7857984005278301e+09 _runtime3113.350806801 _step556լ2- _wandbruntime3113  _runtime31134J'O VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated20047033 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 trainer/global_step6999 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed120455641934848 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed73916929 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed120171866696704 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated27561397 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed123329732685012 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed126330922008064 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 122285568 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 6143981767540 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated46505579 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed126330922008064 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 120432164 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6159142740992 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated47608430 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 6325766103552 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated117185767956832 _step557 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6144067570664 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 6159055311360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated126331026477056 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed20046790 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated126781432942080 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated120171883736064 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed117185750917472 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121ϯN'O WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated120455666838528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated126331026477056 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed120171866696704 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated123329835527496 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed46505577 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated46505579 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed120455641934848 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed73916929 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed120171866696704 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated27561397 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed123329732685012 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed126330922008064 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 122285568 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 6143981767540 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated46505579 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed126330922008064 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 120432164 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6159142740992 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated47608430 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 6325766103552 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated117185767956832 _step557 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6144067570664 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 6159055311360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated126331026477056 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed20046790 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated126781432942080 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated120171883736064 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed117185750917472 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated20047033 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 trainer/global_step6999 JDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated120189050499072 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed120189033459712 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current257 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated73937122 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated126799563677696 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed117202491089792 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated46512222 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated117202508129152 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated27565334 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 trainer/global_step6999 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 6160022367744 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed46512220 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6159934930944 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated73937122 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6159934930944 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed73927452 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current259 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated126349072866816 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6160022367744 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 6326689722368 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 120449344 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed73927452 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated46512222 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated47615226 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated126349072866816 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6144859240762 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed126348968390656 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 120439672 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed47614967 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed123347350330554 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 6326559055360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2NM T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated126349072866816 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6144859240762 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed126348968390656 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 120439672 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed47614967 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed123347350330554 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 6326559055360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated120189050499072 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed120189033459712 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current257 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated73937122 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated126799563677696 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed117202491089792 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated46512222 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated117202508129152 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated27565334 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 trainer/global_step6999 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 6160022367744 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed46512220 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6159934930944 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated73937122 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6159934930944 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed73927452 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current259 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated126349072866816 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6160022367744 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 6326689722368 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 120449344 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed73927452 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated46512222 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated47615226 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0]:2- _wandbruntime3113  _runtime3113 /loss/FRt2/detection_loss2.4581668494563353 trainer/global_step6999 epoch0 _runtime3113.736737712 0loss/FRt1/assignment_loss3.6234683459348016 /loss/FRt2/assignment_loss2.545302785540882 1loss/SRqqt1/assignment_loss1.378487257020814 /loss/FRt1/detection_loss2.5318641845542604 /loss/FBt1/detection_loss0.7962406307106485 _step559 1loss/FBt2/assignment_loss0.13995278333705738 1loss/SRqqt1/detection_loss2.0481822868543014 0loss/SRqqt2/detection_loss1.010980840246467 .loss/FBt2/detection_loss3.746975837009294 %loss/total_loss21.69151465625116 $ _timestamp1.785798400913904e+09 2loss/SRqqt2/assignment_loss1.3614054596489844 0loss/FBt1/assignment_loss0.0504873959373173'P 1loss/FBt2/assignment_loss0.13995278333705738 1loss/SRqqt1/detection_loss2.0481822868543014 0loss/SRqqt2/detection_loss1.010980840246467 .loss/FBt2/detection_loss3.746975837009294 %loss/total_loss21.69151465625116 $ _timestamp1.785798400913904e+09 2loss/SRqqt2/assignment_loss1.3614054596489844 0loss/FBt1/assignment_loss0.0504873959373173 /loss/FRt2/detection_loss2.4581668494563353 trainer/global_step6999 epoch0 _runtime3113.736737712 0loss/FRt1/assignment_loss3.6234683459348016 /loss/FRt2/assignment_loss2.545302785540882 1loss/SRqqt1/assignment_loss1.378487257020814 /loss/FRt1/detection_loss2.5318641845542604 /loss/FBt1/detection_loss0.7962406307106485 _step559ȥW(:  U disk.drbd1828.out0 disk.sda2.out 84.9765625 cpu13.861904110765819 disk.drbd1931.in0 disk.drbd1828.in0 disk.drbd1550.in0 disk.rbd0.in0$ memory_percent41.23634099960327* proc.memory.percent17.182016372680664 disk.drbd1285.in0$ disk./.usageGB578.3459815979004 disk.drbd1673.in0 disk.drbd1938.out0 disk.rbd1.out 12.4697265625 disk.drbd2078.in0 network.recv422642) disk./.usagePercent32.34413490413246 disk.drbd2041.out0 disk.drbd1289.out0 disk.drbd1285.out0 disk.drbd2118.in0 disk.drbd1356.in0 disk.drbd2111.in0 disk.drbd1552.in0 disk.drbd1584.out0 disk.drbd1965.out0 proc.cpu.threads29 disk.drbd1289.in0! disk.dm-0.out146.47705078125 disk.drbd1673.out0 network.sent 6.409757e+06 disk.sda.in0 disk.drbd1965.in0! proc.memory.rssMB 11260.40625 disk.drbd1938.in0 disk.rbd0.out0 disk.sda1.out0 disk.drbd1510.out0 disk.drbd2078.out0 disk.drbd2118.out0 disk.sda2.in0 disk.drbd1550.out0$ disk.nvme0n1.out146.47705078125 disk.drbd1356.out0 disk.rbd1.in0 disk.drbd1226.out0 disk.drbd1817.in0 disk.drbd1931.out0 disk.sda1.in0 disk.drbd1510.in0 disk.dm-0.in 2.47265625 disk.drbd1817.out0) proc.memory.availableMB 38511.3515625 disk.drbd2041.in0 disk.drbd1552.out0 disk.drbd1226.in0 disk.nvme0n1.in 2.47265625 disk.sda.out 84.9765625 disk.drbd1584.in0 disk.drbd2111.out0M(: V gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts96.163' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent32.05433333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0tSI(:  U disk.drbd1584.in0 disk.nvme0n1.out 147.1015625! proc.memory.rssMB 11260.40625 disk.drbd1285.in0 disk.drbd1356.in0 disk.drbd1931.in0 disk.rbd0.out0 disk.drbd1584.out0 disk.drbd1289.out0 disk.drbd1226.out0 disk.drbd1965.in0 disk.drbd2041.out0 disk.drbd1289.in0 disk.sda2.out85.6875 disk.drbd1817.in0 disk.drbd1673.in0 disk.drbd1550.in0 disk.sda1.out0 network.recv425106 disk.drbd1285.out0 disk.nvme0n1.in 2.47265625 disk.drbd1817.out0 disk.drbd2041.in0 disk.drbd1226.in0% memory_percent41.235512495040894* proc.memory.percent17.182016372680664 disk.dm-0.out 147.1015625 disk.drbd2111.in0 disk.rbd1.out 12.5322265625 proc.cpu.threads29$ disk./.usageGB578.3424835205078 disk.drbd2118.in0 disk.drbd1828.in0 disk.drbd1828.out0 disk.drbd1552.in0 disk.drbd1938.in0 disk.drbd1938.out0 disk.drbd2078.in0 disk.drbd2078.out0 disk.drbd2118.out0 disk.rbd1.in0 disk.rbd0.in0 disk.drbd1550.out0( disk./.usagePercent32.3439392733324 disk.drbd1356.out0 disk.drbd1931.out0 disk.sda.in0 disk.drbd1673.out0 disk.drbd1510.out0 disk.sda2.in0 disk.drbd1552.out0 disk.sda1.in0* proc.memory.availableMB38511.89453125 cpu13.86032494069136 network.sent 6.457012e+06 disk.drbd1510.in0 disk.drbd2111.out0 disk.drbd1965.out0 disk.sda.out85.6875 disk.dm-0.in 2.47265625TJ(: ӬV gpu.0.gpu27.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts96.012' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent32.004 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0w2- _wandbruntime3135  _runtime3135W/ trainer/global_step7049 % _timestamp1.7857984226511662e+09 _runtime3135.474113129 _step560 & lr-AdamW/pg10.0001816681425349227 & lr-AdamW/pg20.0001816681425349227SƟ  & lr-AdamW/pg20.0001816681425349227 trainer/global_step7049 % _timestamp1.7857984226511662e+09 _runtime3135.474113129 _step560 & lr-AdamW/pg10.0001816681425349227;2- _wandbruntime3135  _runtime3135g'O N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 6370964750848 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed127233256293888 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current251 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed20190683 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 % _timestamp1.7857984226521065e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed46837727 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 121282658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated121030221886464 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6203138876416 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed46837727 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated20190932 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 6203138876416 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 6203051446784 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated127233360762880 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated127686987429376 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 6370825688064 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed121030204847104 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated47949179 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed118022759533472 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed127233256293888 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed124210629764520 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated118022776572832 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed27758245 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated127233360762880 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6203051446784 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed127686823462912 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated74454587 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current249 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 121292316 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated46837729 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated74454587 trainer/global_step7049 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed47948928 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 _step561 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed121030204847104 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed127233256293888 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current251 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed20190683 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 % _timestamp1.7857984226521065e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed46837727 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 121282658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated121030221886464 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6203138876416 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed46837727 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated20190932 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 6203138876416 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 6203051446784 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated127233360762880 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated127686987429376 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 6370825688064 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed121030204847104 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated47949179 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed118022759533472 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed127233256293888 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed124210629764520 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated118022776572832 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed27758245 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated127233360762880 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6203051446784 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed127686823462912 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated74454587 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current249 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 121292316 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated46837729 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated74454587 trainer/global_step7049 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed47948928 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 _step561 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed121030204847104 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 6370964750848 B6DeviceStatsMonitor.on_train_batch_ena%d/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated47955854 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated127251407152640 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 _step562 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current250 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6203931066368 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated127705094580224 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed127704932718080 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 6371864784896 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 121309496 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed46844370 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated74465124 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6188833507506 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed127251302676480 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed47955602 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated121333229795328 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 trainer/global_step7049 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 121309496 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed74455454 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current252 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed124228247410062 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed121333204891648 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated121047388649472 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated118039516745152 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated74465124 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 121299824 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated20193670 % _timestamp1.7857984230192173e+09 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed121047371610112 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed121047371610112 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed118039499705792 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed27762182 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed74455454 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6203931066368 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 6371727826432 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6188747704270 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated46844372."&M NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated74465124 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 121299824 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated20193670 % _timestamp1.7857984230192173e+09 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed121047371610112 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed121047371610112 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed118039499705792 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed27762182 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed74455454 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6203931066368 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 6371727826432 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6188747704270 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated46844372 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed20193420 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 6204018503168 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated47955854 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated127251407152640 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 _step562 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current250 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6203931066368 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated127705094580224 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed127704932718080 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 6371864784896 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 121309496 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed46844370 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated74465124 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6188833507506 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed127251302676480 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed47955602 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated121333229795328 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 trainer/global_step7049 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 121309496 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed74455454 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current252 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed124228247410062 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed121333204891648 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated121047388649472 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated118039516745152 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360?72- _wandbruntime3135  _runtime3135 k 0loss/FBt2/assignment_loss0.9994067437946796 1loss/SRqqt2/detection_loss0.8926712511373418 trainer/global_step7049 % _timestamp1.7857984230208642e+09 /loss/FBt1/detection_loss1.2598685684746929 &loss/total_loss20.727240577066425 /loss/FRt1/assignment_loss3.268193135338445 2loss/SRqqt2/assignment_loss1.1398967167945926 1loss/FBt1/assignment_loss0.08312104815145037 0loss/SRqqt1/detection_loss2.270830416253635 /loss/FBt2/detection_loss2.0415313347111805 epoch0 0loss/FRt2/assignment_loss2.3471998780965806 2loss/SRqqt1/assignment_loss1.4720677307673864 .loss/FRt1/detection_loss2.517845960714484 .loss/FRt2/detection_loss2.434607792831957 _runtime3135.843698995 _step563"'7 epoch0 0loss/FRt2/assignment_loss2.3471998780965806 2loss/SRqqt1/assignment_loss1.4720677307673864 .loss/FRt1/detection_loss2.517845960714484 .loss/FRt2/detection_loss2.434607792831957 _runtime3135.843698995 _step563 0loss/FBt2/assignment_loss0.9994067437946796 1loss/SRqqt2/detection_loss0.8926712511373418 trainer/global_step7049 % _timestamp1.7857984230208642e+09 /loss/FBt1/detection_loss1.2598685684746929 &loss/total_loss20.727240577066425 /loss/FRt1/assignment_loss3.268193135338445 2loss/SRqqt2/assignment_loss1.1398967167945926 1loss/FBt1/assignment_loss0.08312104815145037 0loss/SRqqt1/detection_loss2.270830416253635 /loss/FBt2/detection_loss2.0415313347111805C(: ɺU gpu.0.gpu0.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts121.282' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent40.42733333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0VpW(:  U disk.drbd1510.out0 disk.drbd1673.in0 disk.drbd1552.out0 disk.drbd1817.out0 disk.drbd1965.in0 disk.drbd1938.out0 disk.drbd1285.out0 disk.drbd2111.in0* proc.memory.availableMB38511.11328125 disk.rbd1.out 12.5322265625 disk.drbd1552.in0 disk.drbd1550.in0 disk.rbd0.out0 cpu13.858877702339658 disk.rbd1.in0 disk.drbd1965.out0 disk.drbd2041.in0 disk.drbd1828.out0 disk.drbd1550.out0 disk.drbd1356.in0 disk.sda2.in0! proc.memory.rssMB 11260.40625 disk.sda1.in0 disk.drbd2078.in0 disk.drbd1356.out0 network.sent 6.502144e+06! disk.dm-0.out147.89111328125 disk.drbd2118.out0 disk.nvme0n1.in 2.47265625* proc.memory.percent17.182016372680664) disk./.usagePercent32.34415815795712 proc.cpu.threads29 disk.dm-0.in 2.47265625$ disk.nvme0n1.out147.89111328125 disk.drbd1289.in0 disk.sda.in0 disk.drbd1226.in0 disk.drbd1226.out0 disk.drbd1584.in0 disk.drbd1584.out0 network.recv427220$ disk./.usageGB578.3463973999023 disk.drbd1817.in0 disk.drbd1673.out0 disk.sda.out 86.1484375 disk.sda2.out 86.1484375# memory_percent41.2367045879364 disk.drbd1931.in0 disk.drbd1828.in0 disk.drbd2111.out0 disk.drbd1931.out0 disk.drbd1289.out0 disk.drbd1938.in0 disk.sda1.out0 disk.drbd1510.in0 disk.drbd2041.out0 disk.drbd1285.in0 disk.drbd2078.out0 disk.drbd2118.in0 disk.rbd0.in0]F2- _wandbruntime3157  _runtime31571w % lr-AdamW/pg10.000182956751859188 % lr-AdamW/pg20.000182956751859188 trainer/global_step7099 % _timestamp1.7857984447870286e+09 _runtime3157.609978783 _step564 D8 % lr-AdamW/pg10.000182956751859188 % lr-AdamW/pg20.000182956751859188 trainer/global_step7099 % _timestamp1.7857984447870286e+09 _runtime3157.609978783 _step564w2- _wandbruntime3157  _runtime3157"O \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed121888542997504 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated121888560036864 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed128135590579712 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 122152468 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 _runtime3157.613680511 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed27955095 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed74972933 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated128135695048704 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated74982589 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated20333672 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed128592408266752 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6231844497680 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed125091526844028 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed118859768149472 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 6416054651904 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 149286400 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated47169879 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed48288512 trainer/global_step7099 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 6231758694556 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed47169877 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 122142810 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current255 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated118859785188832 _step565 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed122176353614848 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6247135011840 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated48288769 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated122176378518528 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated125091629686512 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed74972933 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated121888560036864 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6247047582208 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 % _timestamp1.7857984447879953e+09 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0*6'O PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed74972933 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated121888560036864 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6247047582208 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 % _timestamp1.7857984447879953e+09 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed121888542997504 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated121888560036864 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed128135590579712 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 122152468 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 _runtime3157.613680511 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed27955095 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed74972933 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated128135695048704 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated74982589 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated20333672 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed128592408266752 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6231844497680 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed125091526844028 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed118859768149472 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 6416054651904 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 149286400 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated47169879 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed48288512 trainer/global_step7099 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 6231758694556 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed47169877 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 122142810 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current255 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated118859785188832 _step565 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed122176353614848 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6247135011840 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated48288769 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated122176378518528 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated1250916296865122- _wandbruntime3157  _runtime3157Zt&M M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated128153741438464 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated74993126 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed74983456 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated125109247332166 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed118876508321792 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 6417090234368 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6232636167778 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 trainer/global_step7099 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed121905709760512 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated27959034 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6247927201792 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6247927201792 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 122159976 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current261 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 122169648 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed125109144489570 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed48295309 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 122169648 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated20336538 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 6248014638592 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 6416955373056 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _runtime3157.989052485 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed27959032 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed47176520 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated122193585635328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6232721971014 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated47176522 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated47176522 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated118876525361152 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated128153741438464 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed128610516104704V9&M K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed48295309 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 122169648 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated20336538 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 6248014638592 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 6416955373056 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _runtime3157.989052485 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed27959032 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed47176520 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated122193585635328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6232721971014 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated47176522 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated47176522 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated118876525361152 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated128153741438464 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed128610516104704 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated128153741438464 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated74993126 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed74983456 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated125109247332166 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed118876508321792 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 6417090234368 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6232636167778 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 trainer/global_step7099 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed121905709760512 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated27959034 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6247927201792 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6247927201792 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 122159976 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current261 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 122169648 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed125109144489570 B2- _wandbruntime3157  _runtime3157\UV /loss/FBt2/detection_loss2.6045740180545383 &loss/total_loss21.002532529883922 _step567 1loss/FBt1/assignment_loss0.10121631606500914 /loss/FRt1/detection_loss2.6799693208142936 /loss/FRt2/detection_loss2.3414557157800746 0loss/SRqqt2/detection_loss1.032921795221853 epoch0 trainer/global_step7099 _runtime3157.990835544 % _timestamp1.7857984451677306e+09 2loss/SRqqt2/assignment_loss1.4356786796629342 0loss/FBt2/assignment_loss0.3288991455782606 0loss/SRqqt1/detection_loss1.985938706258977 /loss/FBt1/detection_loss1.3781412773716606 /loss/FRt1/assignment_loss3.293974765812058 /loss/FRt2/assignment_loss2.481177628804476 1loss/SRqqt1/assignment_loss1.338585160459791l  /loss/FRt1/assignment_loss3.293974765812058 /loss/FRt2/assignment_loss2.481177628804476 1loss/SRqqt1/assignment_loss1.338585160459791 /loss/FBt2/detection_loss2.6045740180545383 &loss/total_loss21.002532529883922 _step567 1loss/FBt1/assignment_loss0.10121631606500914 /loss/FRt1/detection_loss2.6799693208142936 /loss/FRt2/detection_loss2.3414557157800746 0loss/SRqqt2/detection_loss1.032921795221853 epoch0 trainer/global_step7099 _runtime3157.990835544 % _timestamp1.7857984451677306e+09 2loss/SRqqt2/assignment_loss1.4356786796629342 0loss/FBt2/assignment_loss0.3288991455782606 0loss/SRqqt1/detection_loss1.985938706258977 /loss/FBt1/detection_loss1.3781412773716606&6(: U gpu.0.gpu38.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts114.971' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent38.32366666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0mX(:  U$ disk./.usageGB578.3464736938477 disk.drbd1965.in0 disk.drbd1550.out0 disk.drbd1938.in0 disk.drbd2078.in0 disk.drbd1931.in0 disk.drbd1289.out0 disk.rbd0.in0 disk.sda2.out 86.48046875 disk.drbd1285.in0 disk.drbd1552.in0 disk.drbd1584.out0 disk.drbd1356.out0 disk.drbd2111.in0 disk.drbd1931.out0 disk.drbd1828.in0 disk.drbd2118.out0 disk.drbd2111.out0* proc.memory.availableMB38511.04296875! proc.memory.rssMB 11260.40625 disk.drbd2041.out0 disk.sda2.in0) disk./.usagePercent32.34416242471393 disk.drbd1965.out0 disk.drbd1289.in0 disk.sda.out 86.48046875 disk.drbd1226.out0 network.recv428334 disk.drbd1817.out0 disk.rbd1.in0 disk.sda1.in0 disk.sda1.out0 disk.drbd1285.out0 disk.rbd1.out 12.5322265625 disk.drbd1673.out0 disk.drbd2041.in0 disk.drbd1552.out0 disk.drbd1938.out0 disk.sda.in0 disk.drbd2118.in0 proc.cpu.threads29 disk.drbd1550.in0 disk.drbd1510.in0 disk.dm-0.in 2.47265625 disk.drbd1356.in0" memory_percent41.236811876297* proc.memory.percent17.182016372680664 cpu13.857329248800317! disk.dm-0.out148.51806640625 disk.nvme0n1.in 2.47265625 network.sent 6.505206e+06 disk.drbd1673.in0 disk.drbd1510.out0 disk.drbd1226.in0 disk.drbd1828.out0 disk.drbd2078.out0$ disk.nvme0n1.out148.51806640625 disk.drbd1817.in0 disk.rbd0.out0 disk.drbd1584.in0 ~2- _wandbruntime3179  _runtime3179ި & lr-AdamW/pg10.0001842453611834533 & lr-AdamW/pg20.0001842453611834533 trainer/global_step7149 # _timestamp1.78579846683613e+09 _runtime3179.659095183 _step568& & lr-AdamW/pg10.0001842453611834533 & lr-AdamW/pg20.0001842453611834533 trainer/global_step7149 # _timestamp1.78579846683613e+09 _runtime3179.659095183 _step568pVF2- _wandbruntime3179  _runtime3179pc'O L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated129038029334528 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated75510591 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated48629516 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated20477569 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 trainer/global_step7149 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed129037924865536 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 123012620 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed129037924865536 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed123036709454848 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed20477310 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current259 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated129498128618496 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6275732961188 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 6461261488640 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated47502029 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated123036734358528 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed122746881147904 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated122746898187264 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0XrWi'O G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current259 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated129498128618496 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6275732961188 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 6461261488640 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated47502029 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated123036734358528 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed122746881147904 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated122746898187264 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated129038029334528 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated75510591 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated48629516 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated20477569 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 trainer/global_step7149 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed129037924865536 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 123012620 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed129037924865536 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed123036709454848 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed20477310 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251d(: þU gpu.0.gpu27.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts123.774' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent41.258 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0V9P(:  þU disk.sda2.out 86.7734375$ memory_percent41.23718738555908" disk.nvme0n1.out 149.447265625 disk.drbd1356.out0 disk.drbd1817.out0 disk.drbd1552.out0 network.recv430664 disk.sda1.out0 disk.nvme0n1.in 2.47265625! proc.memory.rssMB 11260.40625 disk.drbd1550.in0 disk.sda2.in0 disk.drbd2111.out0 disk.drbd1550.out0 disk.drbd1285.in0 disk.drbd1817.in0 disk.drbd1673.in0 disk.drbd1673.out0 disk.drbd1931.in0* proc.memory.percent17.182016372680664 disk.drbd1931.out0 disk.drbd2041.in0 disk.drbd1828.out0 disk.drbd1938.out0 disk.drbd1510.out0 disk.drbd1356.in0) disk./.usagePercent32.34429384082389 disk.drbd1289.out0 disk.rbd0.out0 disk.sda.in0 disk.dm-0.in 2.47265625 disk.rbd1.in0 disk.drbd2041.out0 disk.drbd1965.in0 disk.drbd1965.out0 disk.drbd1938.in0 disk.sda1.in0 disk.drbd1584.in0 disk.drbd2118.in0 disk.drbd2118.out0 disk.drbd2078.out0 disk.rbd1.out 12.658203125 disk.drbd1552.in0( proc.memory.availableMB 38510.796875 disk.drbd1226.in0$ disk./.usageGB578.3488235473633 disk.drbd1828.in0 disk.drbd2078.in0 disk.drbd2111.in0 disk.sda.out 86.7734375 disk.rbd0.in0 disk.dm-0.out 149.447265625 disk.drbd1226.out0 disk.drbd1584.out0 cpu13.855830671253036 network.sent 6.55029e+06 disk.drbd1289.in0 disk.drbd1285.out0 disk.drbd1510.in0 proc.cpu.threads29] 2- _wandbruntime3180  _runtime3180?M F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed129055971248128 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 123029800 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6276524631286 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed122764047910912 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed129516079051776 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated122764064950272 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 123029800 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated47508672 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed20480165 trainer/global_step7149 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6291923337216 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed48636047 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated20480410 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated129056075724288 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed47508670 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated48636294 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6276610434522 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated122764064950272 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _runtime3180.039518057 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 123020128 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 6462162480128 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated47508672 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated75521128 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed75511458 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed125990041569078 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed119713516937792 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6292010774016 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 6462295244288 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0<78&M =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed129055971248128 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 123029800 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6276524631286 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed122764047910912 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed129516079051776 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated122764064950272 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 123029800 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated47508672 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed20480165 trainer/global_step7149 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6291923337216 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed48636047 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated20480410 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated129056075724288 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed47508670 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated48636294 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6276610434522 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated122764064950272 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _runtime3180.039518057 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 123020128 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 6462162480128 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated47508672 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated75521128 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed75511458 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed125990041569078 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed119713516937792 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6292010774016 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 6462295244288 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 EDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated76038593 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated129940363620352 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 6335039853056 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed129940259151360 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated123605236337664 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 6506568801792 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 123872772 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed48969225 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 123863114 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed123897065294848 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated129940363620352 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed130403507616768 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated123897090198528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 _runtime3202.080996182 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated20620668 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 6506442321920 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 6319535621572 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 trainer/global_step7199 % _timestamp1.7857984892576118e+09 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed129940259151360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed120533785381472 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6319621424696 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed47834177 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated47834179 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated48969465 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated28348797 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed28348795 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed123605219298304 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated76038593 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current240 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed20620430 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6335127282688 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6335039853056 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated120533802420832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640O GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current240 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed20620430 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6335127282688 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6335039853056 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated120533802420832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed123605219298304 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated47834179 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated130403659000320 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated126853423845528 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed76028937 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 _step573 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed47834177 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 6335127282688 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed126853321003044 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated76038593 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated129940363620352 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 6335039853056 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed129940259151360 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated123605236337664 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 6506568801792 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 123872772 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed48969225 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 123863114 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed123897065294848 UGDeviceStatsMoJ?nitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated129940363620352 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed130403507616768 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated123897090198528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 _runtime3202.080996182 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated20620668 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 6506442321920 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 6319535621572 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 trainer/global_step7199 % _timestamp1.7857984892576118e+09 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed129940259151360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed120533785381472 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6319621424696 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed47834177 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated47834179 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated48969465 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated28348797 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed28348795 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed123605219298304 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated76038593 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 :2- _wandbruntime3202  _runtime3202H&M L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6335919472640 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6336006909440 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed76039460 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 123880280 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed20623219 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed120550525553792 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6335919472640 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated129958410010112 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current260 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated130421775883264 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 6507339512320 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6320498898030 _step574 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed47840820 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed48975951 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 % _timestamp1.7857984896339056e+09 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed129958305533952 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated28352734 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed47840820 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated129958410010112 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current262 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed123914272411648 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed28352732 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated123914297315328 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated126871041491182 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 163959296 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6320413094794 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated47840822 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated123622403100672 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 123880280 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 123889952 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated47840822 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130wP&M REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed20623219 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed120550525553792 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6335919472640 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated129958410010112 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current260 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated130421775883264 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 6507339512320 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6320498898030 _step574 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed47840820 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed48975951 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 % _timestamp1.7857984896339056e+09 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed129958305533952 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated28352734 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed47840820 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated129958410010112 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current262 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed123914272411648 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed28352732 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated123914297315328 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated126871041491182 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 163959296 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6320413094794 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated47840822 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated123622403100672 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 123880280 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 123889952 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated47840822 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6335919472640 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6336006909440 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed76039460 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 123880280 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548/ 2- _wandbruntime3202  _runtime32022) /loss/FBt2/detection_loss1.6956082545220852 &loss/total_loss19.827857844844402 2loss/SRqqt2/assignment_loss1.2853826094869873 /loss/FRt2/detection_loss2.6323114672432775 1loss/SRqqt1/detection_loss2.2804578419912747 1loss/SRqqt2/detection_loss1.2369666852871888 _step575 2loss/SRqqt1/assignment_loss1.1674348524323217 /loss/FBt1/detection_loss0.6299483261287345 % _timestamp1.7857984896355429e+09 epoch0 /loss/FRt2/assignment_loss2.588794284972592 1loss/FBt1/assignment_loss0.04397941891705719 1loss/FBt2/assignment_loss0.14034860487896594 .loss/FRt1/detection_loss2.592622146737285 0loss/FRt1/assignment_loss3.5340033522466334 trainer/global_step7199 _runtime3202.458363839_$:  0loss/FRt1/assignment_loss3.5340033522466334 trainer/global_step7199 _runtime3202.458363839 /loss/FBt2/detection_loss1.6956082545220852 &loss/total_loss19.827857844844402 2loss/SRqqt2/assignment_loss1.2853826094869873 /loss/FRt2/detection_loss2.6323114672432775 1loss/SRqqt1/detection_loss2.2804578419912747 1loss/SRqqt2/detection_loss1.2369666852871888 _step575 2loss/SRqqt1/assignment_loss1.1674348524323217 /loss/FBt1/detection_loss0.6299483261287345 % _timestamp1.7857984896355429e+09 epoch0 /loss/FRt2/assignment_loss2.588794284972592 1loss/FBt1/assignment_loss0.04397941891705719 1loss/FBt2/assignment_loss0.14034860487896594 .loss/FRt1/detection_loss2.592622146737285^DV(:  U disk.drbd1817.in0 disk.rbd1.in0 disk.drbd1931.in0 disk.drbd2041.out0 disk.drbd2118.out0 disk.sda2.in0% memory_percent41.235774755477905 disk.drbd1584.out0 disk.drbd1550.in0 disk.drbd1550.out0 disk.drbd1289.in0 disk.drbd1938.out0! disk.dm-0.out152.37451171875 disk.drbd2111.in0 disk.drbd1965.out0 disk.drbd1552.out0 network.recv433088 disk.sda.out 87.453125 disk.drbd1226.in0 network.sent 6.598122e+06 disk.drbd1226.out0 disk.drbd1817.out0 disk.sda.in0 disk.drbd1285.out0$ disk.nvme0n1.out152.37451171875 disk.sda2.out 87.453125 disk.drbd2078.out0 disk.dm-0.in 2.47265625 disk.drbd2118.in0 disk.drbd2111.out0 disk.drbd1828.out0* proc.memory.availableMB38511.72265625* disk./.usagePercent32.344275920445256 disk.rbd0.in0 disk.drbd1965.in0 disk.drbd1584.in0# disk./.usageGB578.348503112793 disk.drbd1285.in0 disk.nvme0n1.in 2.47265625 disk.drbd1356.in0* proc.memory.percent17.182016372680664 proc.cpu.threads29 disk.sda1.out0 disk.drbd1673.in0 disk.drbd1673.out0 disk.drbd2078.in0 disk.drbd2041.in0 disk.rbd0.out0 disk.drbd1356.out0 disk.rbd1.out 12.720703125 disk.drbd1931.out0 cpu13.853031753119335 disk.drbd1289.out0 disk.drbd1938.in0 disk.drbd1510.in0 disk.drbd1510.out0 disk.drbd1552.in0 disk.drbd1828.in0! proc.memory.rssMB 11260.40625 disk.sda1.in0lI(: V gpu.0.gpu24.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts111.362' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent37.120666666666665 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors02- _wandbruntime3223  _runtime3223P|; _runtime3223.779563943 _step576 & lr-AdamW/pg10.0001868225798319839 & lr-AdamW/pg20.0001868225798319839 trainer/global_step7249 % _timestamp1.7857985109566164e+09 % _timestamp1.7857985109566164e+09 _runtime3223.779563943 _step576 & lr-AdamW/pg10.0001868225798319839 & lr-AdamW/pg20.0001868225798319839 trainer/global_step7249a2- _wandbruntime3223  _runtime3223pO H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 6379035988480 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed28545645 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed130842593437184 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated121370811036832 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed121370793997472 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed124463557448704 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed131309051900928 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated127734320925036 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 trainer/global_step7249 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated76566595 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed76556939 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 124732924 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 124723266 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6379123418112 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 6379123418112 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated124757446038528 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed124757421134848 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 YKDeviceStatsMonitor.on_train_bd"qBatch_start/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated76566595 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated20763375 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated130842697906176 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6363509888204 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated130842697906176 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed49308773 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated48166329 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed127734218082552 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6379035988480 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current247 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed20763128 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 6363424085080 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 _runtime3223.780822201 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed127734218082552 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6379035988480 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current247 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed20763128 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 6363424085080 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 _runtime3223.780822201 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 6379035988480 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed28545645 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed130842593437184 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated121370811036832 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed121370793997472 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed124463557448704 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed131309051900928 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated127734320925036 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 trainer/global_step7249 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated76566595 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed76556939 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 124732924 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 124723266 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6379123418112 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 6379123418112 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated124757446038528 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed124757421134848 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated76566595 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated20763375 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated130842697906176 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6363509888204 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated130842697906176 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed49308773 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated48166329 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632ZJGT2- _wandbruntime3224  _runtime3224P&M >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated130860744295936 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated127751938570690 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed127751835728094 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 124750104 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated124480741251072 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 6380003044864 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6379915608064 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6379915608064 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated76577132 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed76567462 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6364387361538 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed49315655 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed131327158190592 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated48172972 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed48172970 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 6552529938944 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6380003044864 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 _step578 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed76567462 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated131327317955584 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated124774653155328 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed124480724211712 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated20766322 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated48172972 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed124480724211712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated49315906 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated130860744295936 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 124740432 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 124750104 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated28549584 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current249 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 % _timestamp1.7857985113304067e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed130860639819776 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 124740432 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6364301558302 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed48172970 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated121387551209152 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251IM LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated20766322 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated48172972 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed124480724211712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated49315906 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated130860744295936 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 124740432 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 124750104 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated28549584 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current249 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 % _timestamp1.7857985113304067e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed130860639819776 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 124740432 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6364301558302 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed48172970 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated121387551209152 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated130860744295936 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated127751938570690 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed127751835728094 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 124750104 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated124480741251072 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 6380003044864 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6379915608064 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6379915608064 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated76577132 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed76567462 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7++ DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6364387361538 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed49315655 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed131327158190592 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated48172972 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed48172970 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 6552529938944 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6380003044864 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 _step578 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed76567462 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated131327317955584 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated124774653155328 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed1244807242117122- _wandbruntime3224  _runtime3224EfH _runtime3224.154992977 _step579 0loss/FBt2/assignment_loss0.2580305196711379 /loss/FBt2/detection_loss1.6453433483839035 trainer/global_step7249 2loss/SRqqt2/assignment_loss1.1994219027625548 /loss/FRt2/detection_loss2.2920959403460053 1loss/SRqqt1/detection_loss2.0704743222844217 /loss/FBt1/detection_loss1.2644819235429168 epoch0 /loss/FRt2/assignment_loss2.555500245642388 2loss/SRqqt1/assignment_loss1.0658766386970397 1loss/FBt1/assignment_loss0.09499425702103705 /loss/FRt1/detection_loss2.7302092482914797 1loss/SRqqt2/detection_loss0.8386179796238055 0loss/FRt1/assignment_loss3.5474445545995557 &loss/total_loss19.562490880866243 % _timestamp1.7857985113321457e+09O.c  /loss/FBt1/detection_loss1.2644819235429168 epoch0 /loss/FRt2/assignment_loss2.555500245642388 2loss/SRqqt1/assignment_loss1.0658766386970397 1loss/FBt1/assignment_loss0.09499425702103705 /loss/FRt1/detection_loss2.7302092482914797 1loss/SRqqt2/detection_loss0.8386179796238055 0loss/FRt1/assignment_loss3.5474445545995557 &loss/total_loss19.562490880866243 % _timestamp1.7857985113321457e+09 _runtime3224.154992977 _step579 0loss/FBt2/assignment_loss0.2580305196711379 /loss/FBt2/detection_loss1.6453433483839035 trainer/global_step7249 2loss/SRqqt2/assignment_loss1.1994219027625548 /loss/FRt2/detection_loss2.2920959403460053 1loss/SRqqt1/detection_loss2.0704743222844217ޚ(: U gpu.0.gpu0.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts127.299' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.43300000000001 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0R(:  U disk.drbd1510.in0 disk.dm-0.in 2.47265625 disk.drbd2078.out0$ disk.nvme0n1.out152.66357421875 disk.drbd1938.in0 disk.sda.in0 disk.drbd2111.out0 disk.drbd1965.out0 disk.drbd1828.out0 disk.drbd2118.out0 network.recv435526 disk.drbd1226.in0 disk.drbd1226.out0 disk.drbd2078.in0 network.sent 6.643147e+06 disk.drbd1938.out0 disk.drbd2118.in0 disk.drbd1828.in0 disk.rbd0.out0! disk.dm-0.out152.66357421875 disk.drbd1584.in0 disk.drbd1550.out0* proc.memory.availableMB38510.23828125 disk.rbd0.in0 disk.drbd1673.out0 disk.drbd2041.in0 disk.drbd2041.out0 disk.drbd1550.in0 disk.sda1.out0 disk.drbd1584.out0 disk.nvme0n1.in 2.47265625 disk.drbd1356.in0! proc.memory.rssMB 11260.40625$ disk./.usageGB578.3490104675293 disk.drbd1289.in0 disk.drbd1285.out0 disk.sda2.out87.71875 disk.drbd1817.out0 disk.drbd1931.out0 disk.drbd1552.in0 disk.drbd1965.in0 cpu13.851610860495791 disk.drbd1289.out0 disk.sda.out87.71875 disk.drbd1285.in0 disk.rbd1.out 12.720703125* proc.memory.percent17.182016372680664 disk.drbd1673.in0 disk.drbd1931.in0 disk.drbd1552.out0 proc.cpu.threads29( disk./.usagePercent32.3443042943781 disk.drbd1510.out0 disk.sda2.in0 disk.drbd2111.in0 disk.sda1.in0 disk.drbd1356.out0 disk.rbd1.in0$ memory_percent41.23803973197937 disk.drbd1817.in0s(: U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts101.433' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent33.811 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0fR(:  U disk.rbd1.out 12.8466796875 disk.drbd1931.in0 disk.sda1.in0 disk.drbd1510.in0! disk.dm-0.out153.16455078125 proc.cpu.threads29 disk.nvme0n1.in 2.47265625 disk.drbd2111.out0 cpu13.850356731435213$ disk.nvme0n1.out153.16455078125 disk.drbd1817.out0 disk.drbd1550.in0 disk.sda.out88.0625) disk./.usagePercent32.34409735667247 disk.drbd1226.out0 disk.drbd2078.in0 disk.drbd1356.out0 disk.drbd1965.out0 disk.rbd0.in0 network.sent 6.643255e+06 network.recv435634 disk.drbd2118.in0 disk.drbd2118.out0 disk.drbd1938.out0 disk.drbd2078.out0 disk.drbd1584.out0 disk.drbd1673.in0 disk.drbd1828.in0 disk.drbd2111.in0 disk.drbd1673.out0 disk.drbd2041.in0 disk.drbd1552.out0 disk.drbd1828.out0 disk.drbd1289.out0 disk.rbd0.out0 disk.sda1.out0 disk.dm-0.in 2.47265625! proc.memory.rssMB 11260.40625* proc.memory.percent17.182016372680664 disk.drbd1931.out0 disk.drbd2041.out0 disk.drbd1550.out0 disk.drbd1289.in0 disk.sda.in0* proc.memory.availableMB38510.52734375$ disk./.usageGB578.3453102111816 disk.rbd1.in0 disk.drbd1584.in0 disk.drbd1356.in0 disk.drbd1965.in0 disk.drbd1285.in0 disk.sda2.in0 disk.sda2.out88.0625 disk.drbd1285.out0 disk.drbd1510.out0 disk.drbd1226.in0 disk.drbd1817.in0 disk.drbd1552.in0 disk.drbd1938.in0$ memory_percent41.23759865760803A*J2- _wandbruntime3245  _runtime3245!- & lr-AdamW/pg10.0001881111891562492 & lr-AdamW/pg20.0001881111891562492 trainer/global_step7299 % _timestamp1.7857985331183937e+09 _runtime 3245.94135077 _step580 p % _timestamp1.7857985331183937e+09 _runtime 3245.94135077 _step580 & lr-AdamW/pg10.0001881111891562492 & lr-AdamW/pg20.0001881111891562492 trainer/global_step7299Hf42- _wandbruntime3245  _runtime3245y'O PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated48498479 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated122207819652832 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated28742497 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 % _timestamp1.7857985331193426e+09 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated125321912638464 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated77094597 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current256 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6407398351712 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed125321895599104 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed48498477 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed28742495 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 6423032123904 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current254 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 6423119553536 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed49649284 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated49649540 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated77094597 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed128615115162060 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 125583418 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed131744927723008 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated20907043 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated128615218004544 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed20906789 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed122207802613472 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed77084941 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 trainer/global_step7299 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated131745032192000 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6423119553536 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed125321895599104 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 6596944200704 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 _runtime3245.942601237 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6423032123904 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated132214746079232 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed131744927723008 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated20907043 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated128615218004544 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed20906789 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed122207802613472 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed77084941 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 trainer/global_step7299 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated131745032192000 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6423119553536 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed125321895599104 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 6596944200704 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 _runtime3245.942601237 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6423032123904 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated132214746079232 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed49649284 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated49649540 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated77094597 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed128615115162060 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 125583418#_H2- _wandbruntime3246  _runtime3246UA M L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed77095464 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed49655991 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed48505120 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated125339079401472 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6423911743488 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated122224559825152 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 _step582 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated125339079401472 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed125339062362112 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated128632835650198 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 XDDeviceStatsMonitor.on_train_bkXatch_end/inactive_split_bytes.all.freed132232697667584 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed48505120 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated28746434 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 trainer/global_step7299 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated48505122 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6423911743488 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed28746432 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed125339062362112 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6423999180288 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed128632732807602 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 6423999180288 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6408275825046 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 125610256 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 125600584 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 125610256 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated28746434 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 trainer/global_step7299 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated48505122 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6423911743488 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed28746432 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed125339062362112 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6423999180288 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed128632732807602 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 6423999180288 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6408275825046 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 125610256 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 125600584 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 125610256 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed49655991 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed48505120 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated125339079401472 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6423911743488 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated122224559825152 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 _step582 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated125339079401472 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed125339062362112 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated128632835650198 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed132232697667584 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121[2- _wandbruntime3246  _runtime3246{Q 0loss/FRt2/assignment_loss2.1244605796437868 1loss/SRqqt1/assignment_loss1.050774587903704 1loss/FBt2/assignment_loss0.13043979369103909 .loss/FRt2/detection_loss2.455343535458538 /loss/FBt1/detection_loss0.9991882368279942 .loss/FBt2/detection_loss1.717125833945083 epoch0 1loss/SRqqt1/detection_loss1.5992873749296581 % _timestamp1.7857985334962506e+09 _runtime3246.319087742 /loss/FRt1/detection_loss2.4244719624519346 1loss/SRqqt2/detection_loss0.9904698817274318 %loss/total_loss18.04496985122539 trainer/global_step7299 _step583 /loss/FRt1/assignment_loss3.451036446435111 2loss/SRqqt2/assignment_loss1.0902610234080408 2loss/FBt1/assignment_loss0.012110594803067394(z,  /loss/FRt1/detection_loss2.4244719624519346 1loss/SRqqt2/detection_loss0.9904698817274318 %loss/total_loss18.04496985122539 trainer/global_step7299 _step583 /loss/FRt1/assignment_loss3.451036446435111 2loss/SRqqt2/assignment_loss1.0902610234080408 2loss/FBt1/assignment_loss0.012110594803067394 0loss/FRt2/assignment_loss2.1244605796437868 1loss/SRqqt1/assignment_loss1.050774587903704 1loss/FBt2/assignment_loss0.13043979369103909 .loss/FRt2/detection_loss2.455343535458538 /loss/FBt1/detection_loss0.9991882368279942 .loss/FBt2/detection_loss1.717125833945083 epoch0 1loss/SRqqt1/detection_loss1.5992873749296581 % _timestamp1.7857985334962506e+09 _runtime3246.319087742(: U gpu.0.gpu37.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts124.57' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent41.523333333333326 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0*Z(:  ɋV disk.drbd1552.out0 disk.drbd2111.out0 disk.rbd1.out 12.8466796875$ memory_percent41.23765230178833 disk.drbd2041.in0 disk.rbd1.in0 disk.drbd2078.in0 disk.sda2.out 88.51171875* proc.memory.percent17.182016372680664 disk.drbd2111.in0 disk.drbd1938.in0 disk.dm-0.in 2.47265625 disk.drbd1550.out0 disk.drbd1289.in0 disk.drbd1938.out0 disk.sda.in0 disk.drbd1226.in0$ disk.nvme0n1.out153.35986328125 disk.drbd1673.in0* disk./.usagePercent32.344311547864685 disk.drbd1931.in0 disk.drbd1550.in0 disk.drbd2078.out0 disk.drbd2118.in0 disk.drbd2118.out0 disk.nvme0n1.in 2.47265625 disk.drbd2041.out0 disk.sda.out 88.51171875 disk.drbd1584.in0! proc.memory.rssMB 11260.40625 disk.drbd1828.in0) proc.memory.availableMB 38510.4921875 disk.sda1.out0 disk.drbd1817.out0 disk.drbd1965.out0 disk.drbd1828.out0 cpu13.848998021986418 disk.drbd1931.out0 disk.drbd1289.out0 disk.rbd0.out0 disk.drbd1285.out0 disk.drbd1510.in0! disk.dm-0.out153.35986328125 proc.cpu.threads29$ disk./.usageGB578.3491401672363 disk.drbd1510.out0 disk.drbd1584.out0 disk.drbd1817.in0 disk.drbd1673.out0 disk.drbd1965.in0 network.sent 6.643699e+06 network.recv436601 disk.drbd1285.in0 disk.drbd1356.in0 disk.rbd0.in0 disk.drbd1226.out0 disk.drbd1356.out0 disk.sda2.in0 disk.sda1.in0 disk.drbd1552.in0)2- _wandbruntime3268  _runtime3268>RV _step584 ' lr-AdamW/pg10.00018939979848051452 ' lr-AdamW/pg20.00018939979848051452 trainer/global_step7349 % _timestamp1.7857985552785752e+09 _runtime3268.101608711 % _timestamp1.7857985552785752e+09 _runtime3268.101608711 _step584 ' lr-AdamW/pg10.00018939979848051452 ' lr-AdamW/pg20.00018939979848051452 trainer/global_step7349l2- _wandbruntime3268  _runtime3268/M 'O ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 6451201012096 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed132647262008832 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 126453228 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed123044811229472 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed77612943 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated132647366477824 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed28939345 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6451286815220 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed126180233749504 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 166063616 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 141159936 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated77622599 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed132647262008832 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated123044828268832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 trainer/global_step7349 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed126180233749504 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6467028259328 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated28939347 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated133120274388992 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed133120108325376 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated132647366477824 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6467115688960 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated48830629 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 6467028259328 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed48830627 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated49988411 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed77612943 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed126478132814848 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed129496012241568 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed21048791 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current275 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 % _timestamp1.7857985552795837e+09 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed49988136 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 6641975510528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated48830629 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current273 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 6642116670464 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0[kO NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated48830629 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current273 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 6642116670464 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes._vl#small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 6451201012096 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed132647262008832 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 126453228 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed123044811229472 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed77612943 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated132647366477824 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed28939345 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6451286815220 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed126180233749504 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 166063616 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 141159936 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated77622599 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed132647262008832 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated123044828268832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 trainer/global_step7349 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed126180233749504 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6467028259328 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated28939347 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated133120274388992 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed133120108325376 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated132647366477824 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6467115688960 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated48830629 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 6467028259328 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed48830627 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated49988411 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed77612943 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed126478132814848 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed129496012241568 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed21048791 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current275 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 % _timestamp1.7857985552795837e+09 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed49988136 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 6641975510528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections02- _wandbruntime3268  _runtime3268a|&M A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed77623466 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed48837270 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current238 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed126495339931648 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed129513629887110 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 trainer/global_step7349 _runtime3268.470560625 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed28943282 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated123061568441152 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed123061551401792 _step586 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated21051922 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 126470408 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 6643013255680 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated48837272 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed49994966 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6452078485318 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated132665412867584 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed133138222520320 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated48837272 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated132665412867584 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 126460736 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated126197417551872 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated49995206 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6452164288554 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated126495364835328 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 % _timestamp1.7857985556471958e+09 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6467907878912 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed21051684 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640w&M GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed48837270 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current238 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed126495339931648 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed129513629887110 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 trainer/global_step7349 _runtime3268.470560625 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed28943282 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated123061568441152 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed123061551401792 _step586 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated21051922 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 126470408 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 6643013255680 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated48837272 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed49994966 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6452078485318 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated132665412867584 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed133138222520320 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated48837272 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated132665412867584 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 126460736 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated126197417551872 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated49995206 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6452164288554 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated126495364835328 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 % _timestamp1.7857985556471958e+09 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6467907878912 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed21051684 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed77623466 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0e62- _wandbruntime3268  _runtime3268! 2loss/SRqqt1/assignment_loss1.7001802956356722 2loss/SRqqt2/assignment_loss1.2297890582724529 -loss/FRt1/detection_loss2.81136432625878 .loss/FRt2/detection_loss2.407243301299797 1loss/SRqqt2/detection_loss1.0108646403669435 /loss/FBt1/detection_loss1.2103716043906438 _runtime3268.471762773 1loss/FBt1/assignment_loss0.10409584045311998 trainer/global_step7349 % _timestamp1.7857985556489186e+09 /loss/FRt2/assignment_loss2.472919128407007 loss/FBt2/assignment_loss0 /loss/SRqqt1/detection_loss2.03938470893156 epoch0 _step587 .loss/FBt2/detection_loss8.764905989170074 %loss/total_loss27.25095524569556 /loss/FRt1/assignment_loss3.499836352509512?  2loss/SRqqt1/assignment_loss1.7001802956356722 2loss/SRqqt2/assignment_loss1.2297890582724529 -loss/FRt1/detection_loss2.81136432625878 .loss/FRt2/detection_loss2.407243301299797 1loss/SRqqt2/detection_loss1.0108646403669435 /loss/FBt1/detection_loss1.2103716043906438 _runtime3268.471762773 1loss/FBt1/assignment_loss0.10409584045311998 trainer/global_step7349 % _timestamp1.7857985556489186e+09 /loss/FRt2/assignment_loss2.472919128407007 loss/FBt2/assignment_loss0 /loss/SRqqt1/detection_loss2.03938470893156 epoch0 _step587 .loss/FBt2/detection_loss8.764905989170074 %loss/total_loss27.25095524569556 /loss/FRt1/assignment_loss3.499836352509512r2(: U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts108.532' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent36.17733333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0T(:  U disk.drbd1673.in0 cpu13.847544330578344 network.sent 6.712461e+06 disk.drbd1550.out0 disk.drbd1289.out0 disk.rbd1.in0 disk.drbd1289.in0 disk.sda1.out0 disk.drbd1510.in0 disk.drbd1584.out0! disk.nvme0n1.out 153.90234375 disk.drbd1965.in0* proc.memory.availableMB38511.01171875* proc.memory.percent17.182016372680664 disk.sda1.in0 disk.drbd1817.in0 disk.drbd2078.in0 disk.drbd2041.in0 disk.drbd2041.out0 disk.sda2.out 88.87890625$ memory_percent41.23685956001282) disk./.usagePercent32.34429768090503 disk.sda.in0 disk.dm-0.in 2.47265625 disk.drbd1828.out0 disk.drbd1285.out0 disk.drbd1226.in0 disk.drbd1938.out0 disk.drbd2111.in0! proc.memory.rssMB 11260.40625 disk.rbd0.in0 disk.drbd1356.out0 disk.rbd1.out 12.9091796875 proc.cpu.threads29 disk.rbd0.out0 disk.drbd1285.in0 disk.dm-0.out 153.90234375 disk.drbd2111.out0 disk.drbd1673.out0 disk.drbd1552.in0 disk.drbd1552.out0 disk.drbd1828.in0 disk.drbd1356.in0 disk.sda2.in0 disk.drbd1931.in0 disk.sda.out 88.87890625 disk.drbd2078.out0 disk.drbd1817.out0 disk.drbd1965.out0 network.recv439665 disk.drbd1550.in0 disk.drbd1938.in0 disk.drbd1510.out0 disk.nvme0n1.in 2.47265625$ disk./.usageGB578.3488922119141 disk.drbd1931.out0 disk.drbd1226.out0 disk.drbd1584.in0 disk.drbd2118.out0 disk.drbd2118.in0R_(: U gpu.0.gpu37.0 gpu.0.memory2, gpu.0.memoryAllocated4.1" 1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts128.145' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent42.715 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0V(:  U disk.dm-0.out 154.865234375 disk.rbd1.out 12.9091796875 disk.drbd1931.in0 disk.drbd2118.out0* proc.memory.percent17.182016372680664 cpu13.846287992661983 disk.drbd1552.out0 disk.sda1.out0 disk.drbd1285.in0* proc.memory.availableMB38510.76171875$ disk./.usageGB578.3491287231445 disk.drbd1931.out0 disk.drbd1828.in0 disk.drbd1550.out0 disk.drbd1289.out0 disk.rbd0.in0 disk.rbd0.out0 disk.drbd2118.in0 disk.drbd1552.in0 disk.drbd1550.in0 proc.cpu.threads29) disk./.usagePercent32.34431090785116 disk.drbd1817.out0 disk.drbd1289.in0 disk.sda1.in0 disk.drbd1510.out0 disk.drbd1226.out0 network.recv439731 disk.drbd1938.out0 disk.dm-0.in 2.47265625 disk.drbd1584.out0 disk.nvme0n1.in 2.47265625 disk.drbd1356.out0 disk.drbd1965.in0 disk.drbd1828.out0 disk.sda.in0 disk.drbd2078.out0 disk.drbd1965.out0 disk.drbd2111.out0 network.sent 6.712527e+06 disk.drbd1817.in0 disk.drbd1673.in0 disk.drbd2041.out0 disk.drbd1285.out0 disk.drbd1226.in0 disk.drbd1584.in0 disk.drbd1356.in0 disk.rbd1.in0 disk.drbd1673.out0 disk.drbd2041.in0 disk.sda.out 89.44921875" disk.nvme0n1.out 154.865234375 disk.sda2.in0! proc.memory.rssMB 11260.40625 disk.drbd1938.in0 disk.drbd2078.in0 disk.sda2.out 89.44921875$ memory_percent41.23724102973938 disk.drbd1510.in0 disk.drbd2111.in08s2- _wandbruntime3290  _runtime3290ۀU & lr-AdamW/pg20.0001906884078047798 trainer/global_step7399 % _timestamp1.7857985773065417e+09 _runtime3290.129527628 _step588 & lr-AdamW/pg10.0001906884078047798N & lr-AdamW/pg10.0001906884078047798 & lr-AdamW/pg20.0001906884078047798 trainer/global_step7399 % _timestamp1.7857985773065417e+09 _runtime3290.129527628 _step588"2- _wandbruntime3290  _runtime32906G 'O PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated49162779 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated130377012163560 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 6687295550464 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated134025809108992 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6495175278728 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated49162779 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed49162777 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated50328966 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed29136195 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current246 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed123881819845472 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 trainer/global_step7399 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated78150601 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated127038588939264 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated133549700763648 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6511111824384 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed49162777 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current248 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated127338513558528 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed78140945 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 6687175362048 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 145092096 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 6495089475604 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed78140945 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed21192523 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated78150601 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated29136197 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed133549596294656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121'O NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed78140945 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 6687175362048 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 145092096 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 6495089475604 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed78140945 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed21192523 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated78150601 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated29136197 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed133549596294656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated49162779 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated130377012163560 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 6687295550464 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated134025809108992 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6495175278728 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated49162779 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed49162777 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated50328966 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed29136195 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current246 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed123881819845472 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 trainer/global_step7399 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated78150601 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated127038588939264 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated133549700763648 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6511111824384 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed49162777 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current248 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated127338513558528 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 2- _wandbruntime3290  _runtime3290Xc&M aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 6688210348544 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed133567642677248 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 _runtime3290.501837388 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed78151468 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated133567747153408 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed127055738662912 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6511991451136 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 6688071292928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated78161138 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated130394629809214 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated49169422 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 _step590 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 127320888 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed49169420 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated127355720675328 trainer/global_step7399 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6511904014336 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated50335815 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current262 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 127330560 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed50335553 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed127355695771648 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated78161138 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated29140134 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated49169422 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed49169420 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6495966948826 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 $ _timestamp1.785798577678509e+09 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated127055755702272 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H?DeviceStatsMonitor.on_train_batch_end>vx/active.small_pool.current9670 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360t&M XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated21195681 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed50335553 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed127355695771648 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated78161138 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated29140134 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated49169422 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed49169420 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6495966948826 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 $ _timestamp1.785798577678509e+09 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated127055755702272 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 6688210348544 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed133567642677248 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 _runtime3290.501837388 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed78151468 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated133567747153408 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed127055738662912 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6511991451136 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 6688071292928 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated78161138 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated130394629809214 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated49169422 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 _step590 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 127320888 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed49169420 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated127355720675328 trainer/global_step7399 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6511904014336 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated50335815 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current262 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 127330560 PDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 6732516608512 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 6538977939112 trainer/global_step7449 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated134452035049472 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6555107959808 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 147189248 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed124718828461472 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated49494929 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed128198844494848 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated134931386007040 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated78678603 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed134451930580480 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated49494929 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 122285568 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed127896910050304 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6555020530176 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed29333045 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0Lı` O L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed78668947 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated127896927089664 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 6555107959808 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 6732394322944 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated78678603 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed134451930580480 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated49494929 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 122285568 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed127896910050304 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6555020530176 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed29333045 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated131257909243068 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated134452035049472 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed131257806400584 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed49494927 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 6732516608512 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 6538977939112 trainer/global_step7449 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated134452035049472 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6555107959808 VHDeviceStzvuatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 147189248 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed124718828461472 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated49494929 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed128198844494848 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated134931386007040 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6539855412334 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 % _timestamp1.7857985998318207e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6555987586560 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 trainer/global_step7449 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated78689140 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated29336984 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed21340505 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed50677487 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6539941215570 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated49501572 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed131275424046126 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 128181040 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed49501570 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6555900149760 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed128216051611648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated131275526888722 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 128181040 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 128190712 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current266 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed49501570 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _runtime3312.655160928 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated134470081439232 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated506777555&M NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated78689140 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated29336984 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed21340505 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed50677487 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6539941215570 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated49501572 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed131275424046126 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 128181040 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed49501570 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6555900149760 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed128216051611648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated131275526888722 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 128181040 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 128190712 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current266 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed49501570 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _runtime3312.655160928 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated134470081439232 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated50677755 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6539855412334 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 % _timestamp1.7857985998318207e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6555987586560 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 trainer/global_step7449I2- _wandbruntime3312  _runtime3312(9 /loss/FBt2/detection_loss2.1849764863458963 %loss/total_loss19.93821091189526 _step595 2loss/SRqqt2/assignment_loss1.0912947520773326 0loss/FBt1/assignment_loss0.1281631321350003 1loss/FBt2/assignment_loss0.11857782140410003 0loss/SRqqt1/detection_loss2.121166262118255 epoch0 _runtime 3312.65637403 2loss/SRqqt1/assignment_loss1.4091941050308592 /loss/FRt1/detection_loss2.6263528892511054 trainer/global_step7449 0loss/SRqqt2/detection_loss1.077014230351363 $ _timestamp1.785798599833525e+09 0loss/FRt1/assignment_loss3.0162084585503686 0loss/FRt2/assignment_loss2.7238631733258565 /loss/FRt2/detection_loss2.3748193752765654 /loss/FBt1/detection_loss1.0665802260285573yt 1loss/FBt2/assignment_loss0.11857782140410003 0loss/SRqqt1/detection_loss2.121166262118255 epoch0 _runtime 3312.65637403 2loss/SRqqt1/assignment_loss1.4091941050308592 /loss/FRt1/detection_loss2.6263528892511054 trainer/global_step7449 0loss/SRqqt2/detection_loss1.077014230351363 $ _timestamp1.785798599833525e+09 0loss/FRt1/assignment_loss3.0162084585503686 0loss/FRt2/assignment_loss2.7238631733258565 /loss/FRt2/detection_loss2.3748193752765654 /loss/FBt1/detection_loss1.0665802260285573 /loss/FBt2/detection_loss2.1849764863458963 %loss/total_loss19.93821091189526 _step595 2loss/SRqqt2/assignment_loss1.0912947520773326 0loss/FBt1/assignment_loss0.1281631321350003(: ʿ䨃U gpu.0.gpu30.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts123.875' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent41.291666666666664 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0q_2O(:  ʿU disk.drbd2041.in0 disk.sda.out 90.515625$ disk./.usageGB578.3461380004883 disk.drbd1673.in0 disk.dm-0.out157.7294921875 disk.drbd1584.in0) disk./.usagePercent32.34414365098394 disk.drbd1931.out0 disk.drbd1550.in0 disk.drbd1289.in0 network.sent 6.763512e+06 disk.drbd1673.out0 disk.rbd0.out0 disk.sda1.out0* proc.memory.percent17.182016372680664 disk.drbd1356.out0 disk.drbd2041.out0 disk.drbd1550.out0 cpu13.843844136275749 disk.sda2.out 90.515625 disk.drbd1817.in0 disk.drbd1931.in0 disk.drbd1938.in0 disk.drbd1285.in0 disk.drbd1285.out0 disk.drbd1552.in0 disk.drbd1828.out0 disk.nvme0n1.in 2.47265625 disk.drbd2111.out0 disk.drbd1817.out0 disk.drbd1289.out0 disk.drbd1510.out0$ memory_percent41.23702049255371! proc.memory.rssMB 11260.40625 proc.cpu.threads29 disk.drbd2078.out0 disk.drbd1356.in0 disk.rbd1.out 13.12109375 disk.drbd1965.out0 disk.rbd0.in0 disk.sda1.in0 disk.drbd2118.out0# disk.nvme0n1.out157.7294921875 disk.drbd2111.in0 disk.drbd1828.in0 disk.drbd1938.out0 disk.dm-0.in 2.47265625' proc.memory.availableMB 38510.90625 disk.drbd1965.in0 disk.sda.in0 disk.drbd1226.out0 disk.drbd2078.in0 disk.drbd1584.out0 network.recv443294 disk.rbd1.in0 disk.drbd1552.out0 disk.drbd1510.in0 disk.drbd1226.in0 disk.drbd2118.in0 disk.sda2.in0kʛ(: ٿ齏U gpu.0.gpu35.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts97.714' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent32.571333333333335 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors06X(:  ٿU* proc.memory.availableMB38511.16015625 cpu13.842767479522283 disk.drbd2041.out0 network.sent 6.808696e+06 disk.drbd1817.in0 disk.drbd1828.out0* proc.memory.percent17.182016372680664 disk.drbd2078.in0 disk.rbd1.in0 disk.drbd1673.in0 disk.rbd0.out0 disk.sda.in0 disk.drbd1938.in0! disk.dm-0.out159.26806640625) disk./.usagePercent32.34423047948516 disk.nvme0n1.in 2.47265625 disk.drbd1285.in0 disk.drbd1285.out0$ disk.nvme0n1.out159.26806640625 proc.cpu.threads29 disk.sda1.in0 disk.drbd1584.in0 disk.drbd1289.in0 disk.drbd1289.out0 disk.drbd2078.out0 disk.drbd1584.out0 disk.drbd2118.out0 disk.drbd2111.in0 disk.drbd2111.out0 disk.drbd1931.in0 disk.drbd1510.out0 disk.drbd2118.in0 disk.drbd2041.in0! proc.memory.rssMB 11260.40625 disk.sda.out 90.94921875 disk.drbd1550.out0 disk.drbd1938.out0 disk.drbd1356.in0 disk.drbd1931.out0 disk.drbd1828.in0 disk.drbd1356.out0 disk.drbd1550.in0 disk.drbd1226.in0 disk.drbd1965.out0 disk.sda1.out0 disk.drbd1226.out0 disk.drbd1965.in0 disk.drbd1552.in0 network.recv445783$ disk./.usageGB578.3476905822754 disk.drbd1510.in0 disk.dm-0.in 2.47265625 disk.sda2.in0 disk.sda2.out 90.94921875 disk.drbd1817.out0 disk.drbd1552.out0 disk.rbd0.in0 disk.rbd1.out 13.26953125 disk.drbd1673.out0$ memory_percent41.23663306236267Ad72- _wandbruntime3334  _runtime3334Z{ _runtime3334.051084004 _step596 ' lr-AdamW/pg10.00019326562645331042 ' lr-AdamW/pg20.00019326562645331042 trainer/global_step7499 % _timestamp1.7857986212280617e+09t _runtime3334.051084004 _step596 ' lr-AdamW/pg10.00019326562645331042 ' lr-AdamW/pg20.00019326562645331042 trainer/global_step7499 % _timestamp1.7857986212280617e+0922- _wandbruntime3334  _runtime3334 O =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed135354264866304 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated128755265240064 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated135354369335296 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _step597 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 129033684 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 % _timestamp1.7857986212290726e+09 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 F@DeviceStatsMon0yitor.on_train_batch_start/reserved_bytes.all.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated135354369335296 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed128755248200704 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated49827079 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed132138703480092 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed49827077 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed79196949 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated128755265240064 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6599104095232 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed29529895 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed125555837077472 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated51013494 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed49827077 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated79206605 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current260 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated125555854116832 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 trainer/global_step7499 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed79196949 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 6777616691712 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated29529897 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated132138806322576 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current258 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed129059200334848 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144'O TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated79206605 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current260 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated125555854116832 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 trainer/global_step7499 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed79196949 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 6777616691712 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated29529897 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated132138806322576 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current258 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed129059200334848 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed135354264866304 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated128755265240064 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated135354369335296 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _step597 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 129033684 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 % _timestamp1.7857986212290726e+09 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated135354369335296 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed128755248200704 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated49827079 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed132138703480092 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed49827077 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed79196949 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated128755265240064 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6599104095232 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed29529895 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed125555837077472 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated51013494 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed49827077 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated02- _wandbruntime3334  _runtime3334)>&M F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current253 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated125572594289152 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed125572577249792 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _runtime3334.416597609 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6599896285184 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed135854925372416 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed135372311248896 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 6778517920768 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6599896285184 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed128772414963712 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated135372415725056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated135855087234560 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed51020003 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6583743875842 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 129050864 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed29533832 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated29533834 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated129076432355328 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 129050864 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated49833722 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _step598 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 % _timestamp1.7857986215932682e+09 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 6599983721984 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed128772414963712 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated79217142 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current251 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated49833722 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _runtime3334.416597609 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6599896285184 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed135854925372416 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed135372311248896 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 6778517920768 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6599896285184 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed128772414963712 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated135372415725056 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated135855087234560 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed51020003 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6583743875842 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 129050864 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed29533832 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated29533834 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated129076432355328 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 129050864 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated49833722 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _step598 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 % _timestamp1.7857986215932682e+09 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 6599983721984 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed128772414963712 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated79217142 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current251 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated49833722 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 PDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 6643100230656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed79724951 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated79734607 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed136256599152128 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 129893836 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated29726747 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed50159227 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6643012801024 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed79724951 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated136742531721216 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated129919581078528 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed126392845693472 _step601 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated50159229 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current255 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 6822819968512 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed133019600559600 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated126392862732832 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed136256599152128 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 trainer/global_step7549 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 6643012801024 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated51353801 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated129613603390464 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed136742376143360 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6626840669252 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 % _timestamp1.7857986429104202e+09 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated133019703402084 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed129613586351104 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 _runtime3355.736659804 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed129613586351104 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated50159229 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current257 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 6626754866128 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed50159227 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed51353544 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 6822950642688 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated79734607 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed129919556174848 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed29726745 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated136256703621120 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated21627054 m'O L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed50159227 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6643012801024 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed79724951 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated136742531721216 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated129919581078528 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed126392845693472 _step601 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated50159229 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current255 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 6822819968512 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed133019600559600 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated126392862732832 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed136256599152128 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 trainer/global_step7549 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 6643012801024 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated51353801 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated129613603390464 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed136742376143360 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6626840669252 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 % _timestamp1.7857986429104202e+09 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated133019703402084 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed129613586351104 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 _runtime3355.736659804 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed129613586351104 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated50159229 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current257 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 6626754866128 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed50159227 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed51353544 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 6822950642688 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated79734607 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed129919556174848 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed29726745 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated136256703621120 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated21627054 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed21626799 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 6643100230656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed79724951 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated79734607 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed136256599152128 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 129893836 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated29726747 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658|2- _wandbruntime3356  _runtime3356Ho!M EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _step602 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current254 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 6823723188224 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 129901344 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 6823851758080 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6643892420608 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated129630770153472 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6643979857408 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated79745144 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed50165870 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed129630753114112 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed129630753114112 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed29730682 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 trainer/global_step7549 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated136274750010880 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated136760639953408 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated129936788195328 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed129936763291648 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed133037218205142 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed50165870 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated129630770153472 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated133037321047738 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated50165872 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated126409602905152 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed51360321 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed79735474 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6627632339350 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated79745144 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed136274645534720 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed79735474 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 129901344 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated136274750010880 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current256 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 _runtime3356.107862502 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed136760486479872 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed126409585865792 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 $ _timestamp1.785798643284264e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated21629893 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed21629639 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 129911016 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 129911016 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated51360577 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated29730684 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0N&M TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 _runtime3356.107862502 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed136760486479872 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed126409585865792 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 $ _timestamp1.785798643284264e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated21629893 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed21629639 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 129911016 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 129911016 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated51360577 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated29730684 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _step602 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current254 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 6823723188224 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 129901344 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 6823851758080 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6643892420608 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated129630770153472 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6643979857408 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated79745144 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed50165870 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed129630753114112 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed129630753114112 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed29730682 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 trainer/global_step7549 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated136274750010880 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated136760639953408 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated129936788195328 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed129936763291648 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed133037218205142 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed50165870 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated129630770153472 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated133037321047738 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated50165872 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated126409602905152 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed51360321 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed79735474 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6627632339350 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated79745144 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed136274645534720 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed79735474 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 129901344 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated136274750010880 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current256 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136j02- _wandbruntime3356  _runtime3356L"э _step603 2loss/SRqqt1/assignment_loss1.3211676398706105 1loss/FBt1/assignment_loss0.06617329363508695 /loss/FBt1/detection_loss0.8376891638326924 _runtime3356.108976141 0loss/FRt1/assignment_loss3.0003565813039805 0loss/FRt2/assignment_loss2.3432580345793617 1loss/FBt2/assignment_loss0.09375742586494848 /loss/FRt1/detection_loss2.6526417430345113 1loss/SRqqt2/detection_loss0.9586322437171921 trainer/global_step7549 % _timestamp1.7857986432861342e+09 /loss/FRt2/detection_loss2.4513406591324864 &loss/total_loss19.231883108215293 epoch0 0loss/SRqqt2/assignment_loss1.01409484589461 1loss/SRqqt1/detection_loss2.0309265920271478 /loss/FBt2/detection_loss2.4618448853226647=w  1loss/SRqqt1/detection_loss2.0309265920271478 /loss/FBt2/detection_loss2.4618448853226647 _step603 2loss/SRqqt1/assignment_loss1.3211676398706105 1loss/FBt1/assignment_loss0.06617329363508695 /loss/FBt1/detection_loss0.8376891638326924 _runtime3356.108976141 0loss/FRt1/assignment_loss3.0003565813039805 0loss/FRt2/assignment_loss2.3432580345793617 1loss/FBt2/assignment_loss0.09375742586494848 /loss/FRt1/detection_loss2.6526417430345113 1loss/SRqqt2/detection_loss0.9586322437171921 trainer/global_step7549 % _timestamp1.7857986432861342e+09 /loss/FRt2/detection_loss2.4513406591324864 &loss/total_loss19.231883108215293 epoch0 0loss/SRqqt2/assignment_loss1.01409484589461- W(:  U network.recv448068 disk.drbd1552.in0 disk.drbd1828.in0 disk.drbd1285.out0! disk.dm-0.out161.40283203125 disk.sda2.in0) disk./.usagePercent32.34421895924176 disk.drbd1673.out0 disk.rbd0.out0 disk.drbd2118.in0 disk.drbd1552.out0 disk.drbd1550.out0 disk.sda1.in0 disk.rbd1.in0 disk.drbd1356.in0 disk.drbd1828.out0 disk.dm-0.in 2.47265625 cpu13.840495174608778 disk.drbd1938.out0 disk.drbd1226.in0 proc.cpu.threads29# disk./.usageGB578.347484588623 disk.rbd1.out 13.42578125 disk.drbd1673.in0 disk.sda1.out0! proc.memory.rssMB 11260.40625 disk.drbd1289.in0 disk.drbd1584.out0 network.sent 6.853824e+06 disk.drbd1550.in0 disk.sda.out 91.44921875 disk.drbd1226.out0 disk.drbd1356.out0 disk.drbd2111.out0 disk.rbd0.in0 disk.drbd1510.in0 disk.drbd2111.in0 disk.drbd1965.in0 disk.drbd2041.in0 disk.drbd1289.out0 disk.drbd1584.in0 disk.drbd2118.out0 disk.nvme0n1.in 2.47265625 disk.drbd1817.in0 disk.drbd1817.out0 disk.drbd1965.out0 disk.drbd2078.in0 disk.drbd2078.out0$ memory_percent41.23677611351013* proc.memory.percent17.182016372680664 disk.drbd1938.in0 disk.sda.in0$ disk.nvme0n1.out161.40283203125 disk.drbd1931.in0 disk.drbd2041.out0 disk.drbd1285.in0 disk.sda2.out 91.44921875 disk.drbd1931.out0 disk.drbd1510.out0* proc.memory.availableMB38511.06640625H2`(: ǜU gpu.0.gpu24.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts117.247' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent39.08233333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0f(: U gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts121.771' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent40.590333333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0.7X(:  U disk.drbd1673.in0 disk.drbd1584.in0* proc.memory.availableMB38511.06640625! proc.memory.rssMB 11260.40625 disk.nvme0n1.in 2.47265625 disk.sda2.in0 disk.drbd1817.in0 cpu13.839049377197169 network.recv450394 disk.rbd1.out 13.42578125 disk.drbd1828.in0 disk.dm-0.in 2.47265625 disk.drbd2118.in0 disk.drbd1285.out0 network.sent 6.901496e+06) disk./.usagePercent32.34420829234971 disk.drbd2041.in0 disk.drbd1938.in0 disk.rbd0.out0 disk.drbd1931.out0 disk.drbd1510.out0 disk.drbd2078.in0 disk.drbd1356.out0 disk.drbd1552.in0! disk.dm-0.out162.36376953125 disk.drbd1550.out0 disk.drbd1938.out0 disk.drbd1226.in0 disk.drbd1584.out0 disk.sda1.in0 disk.sda2.out 92.03515625 disk.drbd1510.in0$ memory_percent41.23677611351013 disk.drbd1289.out0 disk.rbd0.in0$ disk.nvme0n1.out162.36376953125 disk.drbd1356.in0 disk.drbd1817.out0$ disk./.usageGB578.3472938537598 disk.drbd1673.out0 disk.drbd1828.out0 disk.sda1.out0 disk.drbd2111.out0* proc.memory.percent17.182016372680664 disk.drbd1226.out0 disk.rbd1.in0 disk.drbd2041.out0 disk.drbd1289.in0 proc.cpu.threads29 disk.drbd1931.in0 disk.drbd1552.out0 disk.drbd1550.in0 disk.sda.out 92.03515625 disk.drbd1285.in0 disk.drbd2118.out0 disk.drbd1965.in0 disk.sda.in0 disk.drbd2078.out0 disk.drbd1965.out0 disk.drbd2111.in0X12- _wandbruntime3377  _runtime3377| trainer/global_step7599 $ _timestamp1.785798665057172e+09 _runtime3377.880151745 _step604 ' lr-AdamW/pg10.00019584284510184105 ' lr-AdamW/pg20.00019584284510184105Syv ' lr-AdamW/pg10.00019584284510184105 ' lr-AdamW/pg20.00019584284510184105 trainer/global_step7599 $ _timestamp1.785798665057172e+09 _runtime3377.880151745 _step604Ap;2- _wandbruntime3377  _runtime3377<8'O KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated137648102936576 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6687096366080 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed127229854309472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed50491377 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed130471924501504 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated51694913 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 6687096366080 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated80262609 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed137158933437952 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated80262609 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 6687008936448 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated50491379 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 % _timestamp1.7857986650580952e+09 _step605 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed51694653 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 6868166018048 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed133900497639108 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated133900600481592 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 trainer/global_step7599 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed130471924501504 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated137159037906944 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated130471941540864 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed50491377 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated29923597 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed137647949455872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 6670643329636 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6687008936448 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current258 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current260 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated137159037906944 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed137158933437952 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated130471941540864 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 60DeviceStatsMonitor.on_train_batch_start/num_ooms0UǀO LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated137648102936576 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6687096366080 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed127229854309472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed50491377 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed130471924501504 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed133900497639108 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated133900600481592 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 trainer/global_step7599 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed130471924501504 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated137159037906944 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated130471941540864 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed50491377 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated29923597 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed137647949455872 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 6670643329636 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6687008936448 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current258 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current260 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated137159037906944 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed137158933437952 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated130471941540864 q!2- _wandbruntime3378  _runtime3378F&M RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6687888556032 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6687975992832 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated127246611521152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated51701644 _runtime3378.258236602 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated137177084296704 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 130771168 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated130797144035328 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed130797119131648 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 6687975992832 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed127246594481792 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed80263476 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated137666212938240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed137176979820544 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed51701379 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated50498022 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated29927534 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6687888556032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 _step606 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed50498020 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated50498022 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 % _timestamp1.7857986654348586e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed80263476 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed29927532 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6671520802858 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current263 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed21773847 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated137177084296704 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 151376384 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 6869068902912 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated130489108303872 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 6868942430208 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 130771168 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed130489091264512 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 130761496 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6687975992832 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated127246611521152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated51701644 _runtime3378.258236602 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated137177084296704 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 130771168 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated130797144035328 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed130797119131648 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 6687975992832 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed127246594481792 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed80263476 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated137666212938240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed137176979820544 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed51701379 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated50498022 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated29927534 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6687888556032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 _step606 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed50498020 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated50498022 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 % _timestamp1.7857986654348586e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed80263476 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed29927532 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6671520802858 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current263 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed21773847 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated137177084296704 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 151376384 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 6869068902912 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated130489108303872 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 6868942430208 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 130771168 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed130489091264512 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 130761496 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 P ˆ(: U gpu.0.gpu24.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts99.606' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent33.202 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0ҋ7T(:  V) disk./.usagePercent32.34421277244437 disk.drbd1931.out0 disk.drbd1552.in0$ memory_percent41.23678207397461 disk.sda1.out0 disk.drbd1226.in0 disk.drbd2041.out0 disk.drbd1938.out0& proc.memory.availableMB 38511.0625 network.sent 6.946629e+06 disk.rbd1.in0 disk.drbd1550.out0 disk.sda.in0 disk.drbd2078.in0 disk.nvme0n1.in 2.47265625 disk.drbd2111.in0 disk.drbd1817.in0 disk.drbd1938.in0 disk.rbd0.out0 disk.drbd1510.out0 disk.drbd2041.in0 disk.drbd2078.out0 disk.sda2.in0 disk.drbd2111.out0 disk.drbd1289.in0 disk.drbd1285.out0 disk.drbd1584.out0$ disk.nvme0n1.out162.67822265625 disk.drbd1356.out0 disk.drbd1673.out0! disk.dm-0.out162.67822265625 disk.drbd2118.in0 disk.drbd2118.out0* proc.memory.percent17.182016372680664 disk.drbd1510.in0 disk.drbd1226.out0 disk.drbd1965.in0 disk.drbd1673.in0 cpu13.837755994340169$ disk./.usageGB578.3473739624023 disk.sda2.out 92.43359375 disk.drbd1552.out0 disk.dm-0.in 2.47265625 disk.drbd1931.in0 disk.drbd1828.out0 disk.drbd1289.out0 disk.sda.out 92.43359375 disk.sda1.in0 disk.drbd1584.in0 disk.drbd1817.out0! proc.memory.rssMB 11260.40625 disk.rbd1.out 13.48828125 disk.drbd1828.in0 disk.drbd1550.in0 disk.drbd1285.in0 disk.drbd1356.in0 disk.drbd1965.out0 disk.rbd0.in0 network.recv452753 proc.cpu.threads29צU2- _wandbruntime3400  _runtime3400V%zs _runtime3400.252825195 _step608 ' lr-AdamW/pg10.00019713145442610633 ' lr-AdamW/pg20.00019713145442610633 trainer/global_step7649 % _timestamp1.7857986874298327e+09 trainer/global_step7649 % _timestamp1.7857986874298327e+09 _runtime3400.252825195 _step608 ' lr-AdamW/pg10.00019713145442610633 ' lr-Ad;M%amW/pg20.00019713145442610633"2- _wandbruntime3400  _runtime3400Ys'O ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated138061372192768 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed80780955 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated134781497561100 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed138553524336640 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed131330262651904 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated80790611 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed138061267723776 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6714617596268 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated131640292758528 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 6731092501504 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated80790611 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated52035214 _runtime3400.257652489 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed131330262651904 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated128066879964832 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed134781394718616 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed52034959 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 6913256481792 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6731092501504 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated50823529 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6731005071872 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 6913387155968 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 131614140 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 6714531793144 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current255 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 131604482 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated138061372192768 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated138553679914496 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed21914514 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated50823529 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated30120447 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed80780955 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated21914767 % _timestamp1.7857986874308512e+09 _step609 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed50823527 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated131330279691264 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 6731005071872 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 trainer/global_step7649 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated128066879964832 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed134781394718616 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed52034959 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 6913256481792 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6731092501504 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated50823529 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6731005071872 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 6913387155968 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 131614140 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 6714531793144 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current255 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 131604482 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated138061372192768 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated138553679914496 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed21914514 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated50823529 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated30120447 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed80780955 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated21914767 % _timestamp1.7857986874308512e+09 _step609 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed50823527 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated131330279691264 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 6731005071872 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 trainer/global_step7649 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed138061267723776 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6714617596268 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated131640292758528 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 6731092501504 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated80790611 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated52035214 _runtime3400.257652489 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed131330262651904 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed50830170 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed131347429414912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated128083620137152 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 131621648 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed80791478 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated131657499875328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed138079314106368 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated134799115206754 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed131347429414912 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 6914290122752 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6715495069602 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed52041770 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 131631320 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated50830172 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated80801148 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed138571630233088 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6715409266366 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 trainer/global_step7649 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated131347446454272 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed30124382 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed21917388 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed134799012364158 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated50830172 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 6731972128256 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated21917637 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed131657474971648 _runtime 3400.62677433 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6731884691456 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 6914155261440 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed128083603097792 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated138079418582528 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251"M \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6731972128256 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed50830170 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed131347429414912 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated128083620137152 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 131621648 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed80791478 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated131657499875328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed138079314106368 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated134799115206754 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed131347429414912 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 6914290122752 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6715495069602 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed52041770 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 131631320 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated50830172 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated80801148 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed138571630233088 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6715409266366 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 trainer/global_step7649 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated131347446454272 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed30124382 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed21917388 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed134799012364158 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated50830172 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 6731972128256 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated21917637 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed131657474971648 _runtime 3400.62677433 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6731884691456 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 6914155261440 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed128083603097792 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated138079418582528 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251ARs2- _wandbruntime3400  _runtime3400+'מ 2loss/FBt2/assignment_loss0.017167481172484143 trainer/global_step7649 % _timestamp1.7857986878052075e+09 _step611 -loss/FRt1/assignment_loss3.0856762085412 0loss/FRt2/assignment_loss2.3899606157993447 2loss/SRqqt1/assignment_loss1.1211743776220828 1loss/FBt1/assignment_loss0.06579787243559541 &loss/total_loss19.979347330469544 epoch0 0loss/SRqqt1/detection_loss1.992625167913502 1loss/SRqqt2/detection_loss0.9816541788765253 .loss/FBt2/detection_loss3.041580244153738 _runtime3400.628042897 2loss/SRqqt2/assignment_loss1.0323410345919024 /loss/FRt1/detection_loss2.6997953959413477 .loss/FRt2/detection_loss2.263187521148002 .loss/FBt1/detection_loss1.288387232273817ȥ 1loss/SRqqt2/detection_loss0.9816541788765253 .loss/FBt2/detection_loss3.041580244153738 _runtime3400.628042897 2loss/SRqqt2/assignment_loss1.0323410345919024 /loss/FRt1/detection_loss2.6997953959413477 .loss/FRt2/detection_loss2.263187521148002 .loss/FBt1/detection_loss1.288387232273817 2loss/FBt2/assignment_loss0.017167481172484143 trainer/global_step7649 % _timestamp1.7857986878052075e+09 _step611 -loss/FRt1/assignment_loss3.0856762085412 0loss/FRt2/assignment_loss2.3899606157993447 2loss/SRqqt1/assignment_loss1.1211743776220828 1loss/FBt1/assignment_loss0.06579787243559541 &loss/total_loss19.979347330469544 epoch0 0loss/SRqqt1/detection_loss1.992625167913502@D(: ͼU gpu.0.gpu24.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts122.422' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent40.80733333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0uBsW(:  U disk.sda.in0 disk.drbd1356.in0 disk.drbd2041.out0 disk.drbd1552.in0 disk.drbd1828.out0 proc.cpu.threads29$ disk./.usageGB578.3473739624023 disk.drbd1584.in0 disk.drbd1817.out0 disk.drbd1552.out0 disk.drbd1285.out0 disk.drbd1510.in0 disk.drbd1226.out0 disk.nvme0n1.in 2.47265625$ disk.nvme0n1.out162.82275390625 disk.drbd1285.in0 disk.sda2.in0 disk.drbd1965.in0 disk.sda.out 92.73828125 disk.drbd2118.out0) proc.memory.availableMB 38510.7109375 disk.rbd1.in0 disk.rbd0.out0 disk.sda1.in0! disk.dm-0.out162.82275390625 disk.drbd1226.in0 disk.drbd2111.out0* proc.memory.percent17.182016372680664 disk.drbd1965.out0 disk.drbd1673.in0 disk.dm-0.in 2.47265625! proc.memory.rssMB 11260.40625 disk.drbd1931.out0 disk.drbd1550.out0 disk.drbd2111.in0 disk.drbd1938.out0 disk.drbd2118.in0 disk.drbd1356.out0 network.recv452819 disk.drbd1828.in0 cpu13.836334654474712 disk.drbd1673.out0 disk.drbd1550.in0 disk.drbd1510.out0 disk.drbd2078.in0$ memory_percent41.23731851577759 network.sent 6.946695e+06 disk.drbd2041.in0 disk.drbd1817.in0 disk.rbd1.out 13.48828125 disk.drbd1931.in0 disk.drbd1289.out0 disk.rbd0.in0 disk.sda1.out0 disk.sda2.out 92.73828125) disk./.usagePercent32.34421277244437 disk.drbd1289.in0 disk.drbd2078.out0 disk.drbd1584.out0 disk.drbd1938.in0tܚ(: U gpu.0.gpu41.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts99.262' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent33.08733333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0QV(:  ĆV disk.drbd2078.out0 disk.drbd1584.out0 disk.drbd2118.out0 disk.drbd2041.out0 disk.rbd0.in0 network.sent 6.991782e+06 disk.drbd1552.in0 disk.drbd1828.in0 disk.drbd1510.out0 disk.dm-0.in 2.47265625 disk.sda2.in0 disk.drbd1510.in0 disk.drbd1226.out0 disk.drbd1356.in0 disk.sda2.out 93.2578125 disk.drbd1938.out0 disk.sda1.in0# memory_percent41.2369430065155 cpu13.83517123789883 disk.nvme0n1.in 2.47265625 disk.sda.out 93.2578125! proc.memory.rssMB 11260.40625 network.recv455046 disk.drbd2118.in0 disk.drbd1356.out0 disk.drbd1817.out0 disk.rbd1.out 13.7001953125 disk.drbd1673.in0 disk.drbd1552.out0 disk.drbd1289.in0* disk./.usagePercent32.344211705755164# disk./.usageGB578.347354888916 disk.drbd1965.out0 disk.drbd2041.in0 disk.sda1.out0 disk.drbd1285.out0$ disk.nvme0n1.out163.44287109375 disk.drbd1828.out0 disk.drbd1550.in0 disk.drbd1938.in0 disk.drbd2078.in0 disk.drbd2111.out0 disk.rbd1.in0 disk.drbd1550.out0 disk.sda.in0* proc.memory.percent17.182016372680664 disk.drbd1931.out0* proc.memory.availableMB38510.95703125 disk.drbd1584.in0 disk.drbd2111.in0 disk.rbd0.out0 proc.cpu.threads29 disk.drbd1965.in0! disk.dm-0.out163.44287109375 disk.drbd1226.in0 disk.drbd1817.in0 disk.drbd1673.out0 disk.drbd1285.in0 disk.drbd1931.in0 disk.drbd1289.out0t<\j2- _wandbruntime3422  _runtime3422)3 trainer/global_step7699 % _timestamp1.7857987091940794e+09 _runtime3422.017040111 _step612 ' lr-AdamW/pg10.00019842006375037167 ' lr-AdamW/pg20.00019842006375037167L# trainer/global_step7699 % _timestamp1.7857987091940794e+09 _runtime3422.017040111 _step612 ' lr-AdamW/pg10.00019842006375037167 ' lr-AdamW/pg20.000198420063750371672- _wandbruntime3422  _runtime3422'O A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated138963706478592 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed138963602009600 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed81308957 _step613 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 132474292 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 6758420256652 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 132464634 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated81318613 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 6958571147264 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated132188617841664 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed138963602009600 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed135662291798124 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 % _timestamp1.7857987091950152e+09 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated128903888580832 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated30317297 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 6775001207296 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated132188617841664 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 trainer/global_step7699 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed81308957 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated138963706478592 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6775001207296 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed139459062070784 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed51155677 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated135662394640608 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated22058238 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed128903871541472 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated52375535 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed22057987 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated139459219745792 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed132500623694848 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated81318613 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed132188600802304 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated132500648598528\!O GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated128903888580832 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated30317297 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 6775001207296 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated132188617841664 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 trainer/global_step7699 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed81308957 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated138963706478592 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6775001207296 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed139459062070784 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed51155677 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated135662394640608 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated22058238 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed128903871541472 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated52375535 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed22057987 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated139459219745792 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed132500623694848 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated81318613 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed132188600802304 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated132500648598528 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated138963706478592 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed138963602009600 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed81308957 _step613 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 132474292 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 6758420256652 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 132464|l=634 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated81318613 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 6958571147264 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated132188617841664 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed138963602009600 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed135662291798124 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 % _timestamp1.7857987091950152e+09 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251c62- _wandbruntime3422  _runtime3422c&M ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated132205784604672 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated51162322 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated22061145 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated132517855715328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated138981752868352 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6775968263680 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 6959475449344 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated81329150 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated81329150 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed138981648392192 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated138981752868352 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated51162322 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated52382379 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 132491472 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated30321234 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 132481800 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed81319480 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed51162320 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed52382124 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed135679909443666 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated139477331164672 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6759383533110 % _timestamp1.7857987095639572e+09 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed132205767565312 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed30321232 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 132491472 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 132481800 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed132517830811648 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121)ꟁ&M S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed138981648392192 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated138981752868352 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated51162322 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated52382379 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 132491472 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated30321234 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 132481800 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed81319480 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed51162320 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed52382124 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed135679909443666 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated139477331164672 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6759383533110 % _timestamp1.7857987095639572e+09 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed132205767565312 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed30321232 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 132491472 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 132481800 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed132517830811648 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated132205784604672 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated51162322 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated22061145 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated132517855715328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated138981752868352 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6775968263680 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 6959475449344 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated81329150 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated81329150c2- _wandbruntime3422  _runtime3422`  2loss/FBt1/assignment_loss0.007069300207693531 _runtime3422.388718169 .loss/FRt1/detection_loss2.588481848622546 1loss/SRqqt1/detection_loss1.8692803588406792 &loss/total_loss21.984883937467504 % _timestamp1.7857987095658896e+09 0loss/FRt1/assignment_loss3.1058069460185957 2loss/SRqqt2/assignment_loss0.9296949148916129 0loss/FBt2/assignment_loss1.3061458613083232 /loss/FRt2/detection_loss2.2997187352106896 /loss/FBt1/detection_loss1.2337339042589581 epoch0 _step615 0loss/FRt2/assignment_loss2.2805627135582913 1loss/SRqqt2/detection_loss0.9312304128547134 .loss/FBt2/detection_loss4.038344351574779 trainer/global_step7699 1loss/SRqqt1/assignment_loss1.394814590120624ʙ  &loss/total_loss21.984883937467504 % _timestamp1.7857987095658896e+09 0loss/FRt1/assignment_loss3.1058069460185957 2loss/SRqqt2/assignment_loss0.9296949148916129 0loss/FBt2/assignment_loss1.3061458613083232 /loss/FRt2/detection_loss2.2997187352106896 /loss/FBt1/detection_loss1.2337339042589581 epoch0 _step615 0loss/FRt2/assignment_loss2.2805627135582913 1loss/SRqqt2/detection_loss0.9312304128547134 .loss/FBt2/detection_loss4.038344351574779 trainer/global_step7699 1loss/SRqqt1/assignment_loss1.394814590120624 2loss/FBt1/assignment_loss0.007069300207693531 _runtime3422.388718169 .loss/FRt1/detection_loss2.588481848622546 1loss/SRqqt1/detection_loss1.8692803588406792fF(:  U disk.drbd1817.in0 disk.drbd2041.out0 disk.drbd1550.in0 disk.drbd1285.out0 disk.dm-0.out 163.69140625 disk.drbd1226.in0 disk.drbd1356.out0 disk.rbd0.out0 disk.drbd1673.in0 disk.drbd2041.in0 disk.sda1.in0! disk.nvme0n1.out 163.69140625 disk.rbd1.in0 disk.drbd1289.out0 disk.drbd1552.out0 disk.drbd1226.out0 disk.drbd2078.out0 disk.drbd1356.in0 disk.drbd2111.out0 disk.drbd1965.in0 disk.drbd1965.out0 disk.sda2.in0 disk.sda2.out93.75 network.recv457682 disk.drbd1289.in0 disk.drbd1938.in0 disk.drbd1584.in0 disk.nvme0n1.in 2.47265625 disk.sda.in0 disk.sda1.out0$ memory_percent41.23629331588745 disk.drbd1552.in0 disk.drbd1550.out0 disk.drbd1285.in0 disk.drbd1584.out0 disk.drbd2111.in0* proc.memory.percent17.182016372680664 disk.drbd1931.in0 disk.drbd1938.out0! proc.memory.rssMB 11260.40625 proc.cpu.threads29* disk./.usagePercent32.344208505687554# disk./.usageGB578.347297668457 disk.rbd1.out 13.7001953125 disk.drbd2118.out0) proc.memory.availableMB 38511.3828125 network.sent 7.03951e+06 disk.drbd1817.out0 disk.drbd1931.out0 disk.drbd1510.out0 disk.drbd2078.in0 disk.drbd2118.in0 disk.drbd1673.out0 disk.sda.out93.75 disk.drbd1510.in0 disk.dm-0.in 2.47265625 disk.rbd0.in0 disk.drbd1828.in0 disk.drbd1828.out0 cpu13.833951840868567Zp(: V gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts132.426' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent44.141999999999996 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0A2- _wandbruntime3444  _runtime3444R? ' lr-AdamW/pg20.00019970867307463695 trainer/global_step7749 $ _timestamp1.785798731425438e+09 _runtime3444.248395906 _step616 ' lr-AdamW/pg10.00019970867307463695y  ' lr-AdamW/pg10.00019970867307463695 ' lr-AdamW/pg20.00019970867307463695 trainer/global_step7749 $ _timestamp1.785798731425438e+09 _runtime3444.248395906 _step616 ]u2- _wandbruntime3444  _runtime3444;VO YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated139866040764416 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7003793427968 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated129740897196832 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated81846615 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current255 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed139865936295424 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current257 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 _runtime3444.249638272 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6819084772352 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 trainer/global_step7749 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6818997342720 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed133360979534848 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6802394523284 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed51487827 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 166063616 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed129740880157472 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 6818997342720 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated30514147 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated81846615 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed136543188877632 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated133046955992064 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed81836959 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 _step617 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 % _timestamp1.7857987314263616e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed22201264 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed51487827 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 141159936 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed30514145 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed140364631802880 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated136543291720116 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed133046938952704 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peakʌu|  2645753856 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 6802308720160 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 133324786 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed139865936295424 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed81836959 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 7003652268032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated52715666 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated51487829 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated133361004438528 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 6819084772352 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 133334444 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated22201519 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated139866040764416 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed52715409 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed133046938952704 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated52715666 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated51487829 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated133361004438528 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 6819084772352 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 133334444 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated22201519 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated139866040764416 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed52715409 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed133046938952704 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed139865936295424 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current257 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 _runtime3444.249638272 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6819084772352 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 trainer/global_step7749 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6818997342720 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed133360979534848 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6802394523284 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed51487827 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 166063616 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed129740880157472 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 6818997342720 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated30514147 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated81846615 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed136543188877632 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated133046955992064 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed81836959 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 _step617 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 % _timestamp1.7857987314263616e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed22201264 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed51487827 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 141159936 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed30514145 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed140364631802880 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated136543291720116 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136a2- _wandbruntime3444  _runtime3444?K&M D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 133341952 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed139883982678016 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6819876962304 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed133378186651648 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed52722150 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated30518084 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated139884087154176 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7004684549120 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed51494470 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed81847482 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 133341952 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated52722379 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated136560909365770 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated51494472 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current229 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated81857152 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6819964399104 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 trainer/global_step7749 % _timestamp1.7857987317890408e+09 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated81857152 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated133064122755072 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 133351624 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed22204068 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated133378211555328 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed133064105715712 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 133351624 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed22204068 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated133378211555328 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed133064105715712 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed52722150 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated30518084 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated139884087154176 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7004684549120 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed51494470 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed81847482 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 133341952 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated52722379 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated136560909365770 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated51494472 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current229 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated81857152 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 E?DeviceStatsMonitor.on_train_batch_end/Txeoversize_segments.current0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6819964399104 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 trainer/global_step7749 % _timestamp1.7857987317890408e+09 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated81857152bZc2- _wandbruntime3444  _runtime3444j /loss/FRt1/assignment_loss2.981009027931128 /loss/FRt2/assignment_loss2.443615386325322 2loss/SRqqt2/assignment_loss0.9626431757460038 /loss/FRt1/detection_loss2.5179784177394398 /loss/FRt2/detection_loss2.4502024047830133 &loss/total_loss19.275138430967363 _step619 2loss/SRqqt1/assignment_loss1.5342727561453076 1loss/SRqqt1/detection_loss1.8153576455784566 .loss/FBt1/detection_loss1.557373211512135 epoch0 # _timestamp1.78579873179079e+09 _runtime3444.613638841 1loss/FBt1/assignment_loss0.06936530436139829 /loss/FBt2/detection_loss1.7904375676531346 1loss/FBt2/assignment_loss0.18617571141803638 1loss/SRqqt2/detection_loss0.9667078217739861 trainer/global_step7749q` 1loss/SRqqt1/detection_loss1.8153576455784566 .loss/FBt1/detection_loss1.557373211512135 epoch0 # _timestamp1.78579873179079e+09 _runtime3444.613638841 1loss/FBt1/assignment_loss0.06936530436139829 /loss/FBt2/detection_loss1.7904375676531346 1loss/FBt2/assignment_loss0.18617571141803638 1loss/SRqqt2/detection_loss0.9667078217739861 trainer/global_step7749 /loss/FRt1/assignment_loss2.981009027931128 /loss/FRt2/assignment_loss2.443615386325322 2loss/SRqqt2/assignment_loss0.9626431757460038 /loss/FRt1/detection_loss2.5179784177394398 /loss/FRt2/detection_loss2.4502024047830133 &loss/total_loss19.275138430967363 _step619 2loss/SRqqt1/assignment_loss1.5342727561453076{O(:  U disk.sda2.in0 disk.drbd1931.in0 disk.drbd1828.out0 disk.drbd1550.in0! proc.memory.rssMB 11260.40625 disk.drbd1817.in0 disk.drbd1673.in0 disk.drbd1938.in0 disk.rbd0.in0 disk.drbd1965.in0 disk.drbd1673.out0 disk.drbd2041.in0 disk.sda.in0 disk.drbd1285.out0 disk.rbd1.out 13.7626953125 disk.drbd2118.in0 disk.drbd1828.in0" disk.nvme0n1.out 164.087890625 disk.drbd1938.out0 proc.cpu.threads29# disk./.usageGB578.347354888916 disk.drbd1584.out0 disk.drbd1550.out0 disk.sda1.in0 disk.drbd2078.in0 disk.drbd1817.out0 disk.drbd1552.out0 disk.sda1.out0 disk.sda2.out94.21875 disk.drbd2111.out0 disk.drbd1285.in0$ memory_percent41.23638868331909* proc.memory.percent17.182016372680664 cpu13.832742242759961 network.sent 7.084604e+06 disk.drbd1289.out0 disk.drbd1226.out0 disk.drbd1931.out0 disk.drbd1289.in0 disk.rbd0.out0 disk.sda.out94.21875 disk.drbd1510.in0) proc.memory.availableMB 38511.3203125 disk.drbd1226.in0 disk.rbd1.in0* disk./.usagePercent32.344211705755164 disk.dm-0.in 2.47265625 disk.dm-0.out 164.087890625 disk.drbd2111.in0 network.recv459422 disk.drbd1510.out0 disk.drbd2118.out0 disk.drbd1584.in0 disk.nvme0n1.in 2.47265625 disk.drbd1356.in0 disk.drbd1356.out0 disk.drbd1552.in0 disk.drbd2078.out0 disk.drbd2041.out0 disk.drbd1965.out0d(: V gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts96.635' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent32.211666666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0SV(:  U disk.drbd2118.in0 disk.drbd1550.in0 disk.drbd1226.out0 disk.drbd1584.out0 disk.rbd1.in0 disk.drbd1965.in0 disk.rbd0.out0 disk.drbd2078.out0 disk.drbd1817.out0 disk.drbd1673.out0 disk.dm-0.out 164.216796875! proc.memory.rssMB 11260.40625 disk.rbd0.in0 disk.sda1.in0$ memory_percent41.23638868331909 network.sent 7.084712e+06* disk./.usagePercent32.343991541103414 disk.drbd2111.in0 disk.rbd1.out 13.7626953125 disk.drbd1965.out0 disk.sda1.out0) proc.memory.availableMB 38511.3203125 disk.drbd1289.in0 cpu13.831681035850382 disk.drbd1356.in0 disk.drbd1226.in0 disk.nvme0n1.in 2.47265625 disk.drbd1931.out0 disk.sda.in0 disk.drbd1285.out0 disk.drbd1552.out0 disk.sda.out 94.69921875 disk.drbd1828.in0 disk.drbd1584.in0 disk.drbd2118.out0 network.recv459530 disk.drbd1931.in0 disk.drbd2041.in0 disk.drbd1938.in0 disk.drbd1938.out0 disk.drbd1510.in0 disk.drbd1510.out0 proc.cpu.threads29$ disk./.usageGB578.3434181213379 disk.drbd1356.out0 disk.drbd1817.in0 disk.drbd1289.out0 disk.drbd1285.in0 disk.drbd2078.in0" disk.nvme0n1.out 164.216796875 disk.drbd2041.out0* proc.memory.percent17.182016372680664 disk.sda2.out 94.69921875 disk.drbd2111.out0 disk.drbd1673.in0 disk.drbd1550.out0 disk.dm-0.in 2.47265625 disk.sda2.in0 disk.drbd1552.in0 disk.drbd1828.out0Zm(: çV gpu.0.gpu31.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts128.774' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.92466666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0؂u*2- _wandbruntime3466  _runtime3466k1> ' lr-AdamW/pg20.00020099728239890224 trainer/global_step7799 $ _timestamp1.785798753281919e+09 _runtime3466.104899159 _step620 ' lr-AdamW/pg10.00020099728239890224 $ _timestamp1.785798753281919e+09 _runtime3466.104899159 _step620 ' lr-AdamW/pg10.00020099728239890224 ' lr-AdamW/pg20.00020099728239890224 trainer/global_step7799D2- _wandbruntime3466  _runtime3466'O trainer/global_step7799 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated51819979 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated133905294142464 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated134221360278528 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 _step621 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated140768375050240 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated141270339774464 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed133905277103104 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed133905277103104 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6846282986792 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 7048842530304 _runtime 3466.10613557 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated51819979 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated137424188799624 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed137424085957140 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed130577888773472 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed140768270581248 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6862993478144 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed22345122 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed82364961 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed53056117 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed30710995 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated134221360278528 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 _step621 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated140768375050240 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated141270339774464 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed133905277103104 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed133905277103104 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6846282986792 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 7048842530304 _runtime 3466.10613557 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated51819979 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated137424188799624 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed137424085957140 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed130577888773472 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed140768270581248 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6862993478144 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed22345122 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed82364961 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed53056117 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed30710995 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated30714934 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6863960534528 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 _step622 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed133922443866112 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated51826622 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed53062940 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated51826622 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated133922460905472 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6863873097728 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed82375484 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated53063199 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current257 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated134238567395328 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 7049744742400 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 trainer/global_step7799 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated82385154 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed82375484 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated82385154 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed130594628945792 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current259 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7049879603712 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated140786421440000 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed51826620 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated130594645985152 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6863873097728 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated140786421440000 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed141288287234048 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 134211776 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed51826620 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed137441703602682+&M aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7049879603712 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated140786421440000 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed51826620 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated130594645985152 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6863873097728 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated140786421440000 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed141288287234048 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 134211776 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed51826620 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed137441703602682 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6847074656890 % _timestamp1.7857987536551988e+09 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed140786316963840 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated30714934 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6863960534528 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 _step622 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed133922443866112 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated51826622 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed53062940 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated51826622 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated133922460905472 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6863873097728 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed82375484 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated53063199 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current257 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated134238567395328 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 7049744742400 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 trainer/global_step7799 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated82385154 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed82375484 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated82385154 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed130594628945792 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current259 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0@G 2- _wandbruntime3466  _runtime3466  /loss/FRt1/assignment_loss2.906457558472951 1loss/FBt1/assignment_loss0.01843183540525528 1loss/FBt2/assignment_loss0.14532786369596096 .loss/FBt2/detection_loss2.070990204811096 trainer/global_step7799 /loss/FRt2/assignment_loss2.156527189946756 1loss/SRqqt2/assignment_loss1.086744495811449 .loss/FRt2/detection_loss2.272460821198254 1loss/SRqqt1/detection_loss1.7575957015433232 0loss/SRqqt2/detection_loss1.018369716338136 &loss/total_loss18.322554893235683 epoch0 % _timestamp1.7857987536566763e+09 2loss/SRqqt1/assignment_loss1.1637252169311978 .loss/FRt1/detection_loss2.753641767501831 /loss/FBt1/detection_loss0.9722825215794728 _runtime3466.479665938 _step623ߨ .loss/FBt2/detection_loss2.070990204811096 trainer/global_step7799 /loss/FRt2/assignment_loss2.156527189946756 1loss/SRqqt2/assignment_loss1.086744495811449 .loss/FRt2/detection_loss2.272460821198254 1loss/SRqqt1/detection_loss1.7575957015433232 0loss/SRqqt2/detection_loss1.018369716338136 &loss/total_loss18.322554893235683 epoch0 % _timestamp1.7857987536566763e+09 2loss/SRqqt1/assignment_loss1.1637252169311978 .loss/FRt1/detection_loss2.753641767501831 /loss/FBt1/detection_loss0.9722825215794728 _runtime3466.479665938 _step623 /loss/FRt1/assignment_loss2.906457558472951 1loss/FBt1/assignment_loss0.01843183540525528 1loss/FBt2/assignment_loss0.14532786369596096#'(: U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts124.821' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent41.607 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0#T(:  U disk.drbd1828.in0 disk.sda1.out0 disk.drbd1226.in0# disk.nvme0n1.out164.5751953125 disk.sda2.in0 disk.drbd2041.out0 disk.rbd0.in0 disk.sda1.in0 disk.rbd1.out 13.888671875% memory_percent41.237592697143555 network.recv461496* disk./.usagePercent32.344018208333516 disk.sda.in0 disk.drbd1285.out0 disk.drbd1584.out0 cpu13.830524016987438 disk.drbd1510.in0 disk.drbd1584.in0 disk.drbd1289.out0 disk.drbd2111.in0 disk.drbd1817.out0 disk.drbd1965.in0 disk.drbd1673.in0 disk.drbd1931.in0 disk.rbd1.in0 disk.drbd1550.in0 disk.drbd1938.in0 disk.rbd0.out0 disk.drbd2078.in0 disk.drbd1356.in0 disk.sda2.out 95.0234375 disk.drbd1510.out0! proc.memory.rssMB 11260.40625 disk.drbd2041.in0 disk.sda.out 95.0234375 disk.dm-0.out164.5751953125 disk.drbd2118.in0' proc.memory.availableMB 38510.53125 disk.drbd1550.out0 disk.drbd2118.out0 proc.cpu.threads29 disk.nvme0n1.in 2.47265625 disk.drbd1552.out0 disk.dm-0.in 2.47265625 disk.drbd1226.out0$ disk./.usageGB578.3438949584961 disk.drbd1931.out0 disk.drbd1552.in0 disk.drbd1289.in0 disk.drbd1938.out0 disk.drbd2078.out0* proc.memory.percent17.182016372680664 network.sent 7.129656e+06 disk.drbd1356.out0 disk.drbd2111.out0 disk.drbd1965.out0 disk.drbd1828.out0 disk.drbd1285.in0 disk.drbd1817.in0 disk.drbd1673.out0,,'(2- _wandbruntime3487  _runtime34874z $ _timestamp1.785798775172002e+09 _runtime3487.994960747 _step624 ' lr-AdamW/pg10.00020228589172316757 ' lr-AdamW/pg20.00020228589172316757 trainer/global_step7849~ ' lr-AdamW/pg10.00020228589172316757 ' lr-AdamW/pg20.00020228589172316757 trainer/global_step7849 $ _timestamp1.785798775172002e+09 _runtime3487.994960747 _step6242- _wandbruntime3487  _runtime3487Bj{U'O SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed142175705211392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed138304983036648 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 6907077043200 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed135081691214848 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated141670709336064 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated135081716118528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated131414914428832 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed52152127 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 6890085647176 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 _step625 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed30907845 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed131414897389472 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 6906989613568 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated82902619 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated30907847 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed53397120 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed22489275 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated138305085879132 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7094148865024 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6890171450300 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed82892963 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated82902619 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 135054748 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 7094013996544 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current260 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated52152129 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed82892963 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated134763632292864 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6906989613568 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed141670604867072 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed134763615253504 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated22489535>CO QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current260 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated52152129 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed82892963 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated134763632292864 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6906989613568 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed141670604867072 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed134763615253504 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated22489535 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed142175705211392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed138304983036648 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 6907077043200 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed135081691214848 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated141670709336064 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated135081716118528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated131414914428832 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed52152127 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejꈔections0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 6890085647176 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 _step625 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed30907845 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed131414897389472 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 6906989613568 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated82902619 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated30907847 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed53397120 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed22489275 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated138305085879132 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7094148865024 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6890171450300 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed82892963 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated82902619 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 135054748 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 7094013996544t5Q2- _wandbruntime3488  _runtime34882 &M NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated52158772 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed22492193 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated141688755725824 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 135071928 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated30911784 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed131431637561792 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 135062256 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed142193810722816 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 135062256 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated141688755725824 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 135071928 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed82903486 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed141688651249664 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed52158770 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 163959296 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated52158772 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated131431654601152 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed52158770 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed134780782016512 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed53403975 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated135098923235328 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed82903486 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated134780799055872 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated142193974682112 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6891048923634 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6890963120398 trainer/global_step7849 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated82913156 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current253 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6907869233152 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 % _timestamp1.7857987755452354e+09!e&M O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 135062256 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed142193810722816 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 135062256 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated141688755725824 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 135071928 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed82903486 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed141688651249664 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed52158770 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 163959296 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated52158772 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated131431654601152 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed52158770 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed134780782016512 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed53403975 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated135098923235328 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed82903486 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated134780799055872 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated142193974682112 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6891048923634 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6890963120398 trainer/global_step7849 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated82913156 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current253 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6907869233152 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 % _timestamp1.7857987755452354e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated52158772 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed22492193 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated141688755725824 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 135071928 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated30911784 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed131431637561792BR2- _wandbruntime3488  _runtime3488 1loss/SRqqt2/detection_loss1.1477750081164566 /loss/FBt2/detection_loss1.8402390265837312 epoch0 .loss/FRt2/detection_loss2.397265056059474 %loss/total_loss19.30490978207898 trainer/global_step7849 0loss/FRt2/assignment_loss2.7402413310039613 1loss/FBt1/assignment_loss0.03734945746684522 1loss/SRqqt1/detection_loss1.7135876394635023 /loss/FBt1/detection_loss1.2627922265888436 /loss/FRt1/assignment_loss3.215113506505364 2loss/SRqqt1/assignment_loss1.2301650956966157 % _timestamp1.7857987755469458e+09 _runtime3488.369776785 _step627 2loss/SRqqt2/assignment_loss1.0413834581788826 1loss/FBt2/assignment_loss0.01404121302857586 /loss/FRt1/detection_loss2.66495676338672640S  epoch0 .loss/FRt2/detection_loss2.397265056059474 %loss/total_loss19.30490978207898 trainer/global_step7849 0loss/FRt2/assignment_loss2.7402413310039613 1loss/FBt1/assignment_loss0.03734945746684522 1loss/SRqqt1/detection_loss1.7135876394635023 /loss/FBt1/detection_loss1.2627922265888436 /loss/FRt1/assignment_loss3.215113506505364 2loss/SRqqt1/assignment_loss1.2301650956966157 % _timestamp1.7857987755469458e+09 _runtime3488.369776785 _step627 2loss/SRqqt2/assignment_loss1.0413834581788826 1loss/FBt2/assignment_loss0.01404121302857586 /loss/FRt1/detection_loss2.6649567633867264 1loss/SRqqt2/detection_loss1.1477750081164566 /loss/FBt2/detection_loss1.8402390265837312QtVR(:  U disk.drbd2041.in0 disk.sda.in0 disk.nvme0n1.in 2.47265625 disk.sda2.out 95.42578125 disk.rbd0.in0 disk.rbd0.out0 disk.drbd1552.out0 disk.drbd1289.in0 disk.sda2.in0 disk.drbd1550.out0 disk.dm-0.in 2.47265625 disk.drbd1965.out0 disk.drbd2041.out0 disk.sda1.out0 disk.drbd1510.in0 disk.rbd1.in0$ disk./.usageGB578.3438377380371 disk.sda1.in0 disk.drbd1285.in0 disk.drbd1510.out0 disk.dm-0.out164.7509765625 disk.drbd2118.out0 network.sent 7.129764e+06 disk.drbd1931.in0 disk.drbd1550.in0 disk.drbd1817.in0* proc.memory.percent17.182016372680664 disk.drbd2078.out0 disk.rbd1.out 13.888671875 proc.cpu.threads29 disk.drbd1931.out0 disk.drbd1828.out0 disk.drbd1226.out0 disk.drbd2078.in0 disk.drbd1584.out0 disk.drbd1817.out0 disk.drbd1673.in0 disk.drbd1673.out0 cpu13.829377662236828 disk.drbd1938.in0 disk.drbd2111.in0$ memory_percent41.23725891113281 disk.drbd1552.in0 disk.drbd1226.in0# disk.nvme0n1.out164.7509765625 disk.drbd1965.in0 disk.drbd1828.in0 disk.drbd2118.in0 network.recv461604 disk.drbd1584.in0 disk.drbd1938.out0 disk.drbd1356.in0 disk.drbd2111.out0* disk./.usagePercent32.344015008265906 disk.drbd1289.out0 disk.sda.out 95.42578125 disk.drbd1285.out0 disk.drbd1356.out0$ proc.memory.availableMB38510.75! proc.memory.rssMB 11260.40625y{(: U gpu.0.gpu23.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts131.255' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent43.751666666666665 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0Q+(: U gpu.0.gpu4.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts95.846' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.948666666666668 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0ʕV(:  U disk.rbd0.in0 disk.sda.out 95.7734375 disk.drbd1226.in0 disk.drbd1584.out0 disk.drbd2111.out0 disk.drbd1289.out0 disk.drbd1356.in0 disk.rbd1.in0 disk.drbd2118.in0 disk.drbd1965.out0 disk.drbd2041.out0) proc.memory.availableMB 38510.4921875 disk.sda.in0 disk.drbd1510.out0 disk.drbd1931.in0 disk.drbd1552.in0 disk.drbd1552.out0 disk.drbd1550.in0 disk.drbd1938.in0 disk.dm-0.in 2.47265625 disk.rbd1.out 13.951171875 proc.cpu.threads29! proc.memory.rssMB 11260.40625 disk.drbd1828.out0$ disk.nvme0n1.out165.05029296875 disk.sda2.in0 disk.sda2.out 95.7734375 disk.drbd1817.out0 disk.sda1.in0 disk.drbd1673.out0* proc.memory.percent17.182016372680664 disk.drbd2118.out0 disk.drbd1965.in0 disk.drbd2041.in0 disk.drbd1817.in0 disk.sda1.out0 disk.drbd1510.in0 disk.drbd2078.out0 cpu13.828241613527458 disk.drbd1285.in0 disk.drbd1226.out0 disk.drbd1931.out0 network.sent 7.177363e+06 disk.drbd1828.in0 disk.drbd1584.in0 disk.drbd1356.out0 disk.drbd1673.in0 disk.drbd1938.out0! disk.dm-0.out165.05029296875 disk.drbd2111.in0$ memory_percent41.23765230178833 disk.drbd1289.in0 disk.rbd0.out0 disk.drbd1285.out0 disk.nvme0n1.in 2.47265625 network.recv463560 disk.drbd2078.in0) disk./.usagePercent32.34401714164431$ disk./.usageGB578.3438758850098 disk.drbd1550.out09W2- _wandbruntime3510  _runtime3510i^ ' lr-AdamW/pg10.00020357450104743286 ' lr-AdamW/pg20.00020357450104743286 trainer/global_step7899 % _timestamp1.7857987974429483e+09 _runtime3510.265939987 _step628MA0 ' lr-AdamW/pg20.00020357450104743286 trainer/global_step7899 % _timestamp1.7857987974429483e+09 _runtime3510.265939987 _step628 ' lr-AdamW/pg10.00020357450104743286'q2- _wandbruntime3510  _runtime3510O BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed142572939152896 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated52484279 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6951073178624 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated53736975 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed132251906005472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed83420965 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed52484277 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 6933974110684 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated142573043621888 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 6951073178624 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current243 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed22632035 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed143081194615296 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed135942047054848 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6934059913808 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed52484277 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated83430621 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed135621953403904 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed53736730 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated83430621 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated135621970443264 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated52484279 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated135942071958528 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 % _timestamp1.7857987974439344e+09 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed139185880116156 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 _runtime3510.270661549 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 135905242 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7139276137472 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 135914900 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated31104697 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed142572939152896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6950985748992 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6934059913808 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed52484277 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated83430621 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed135621953403904 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed53736730 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated83430621 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated135621970443264 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated52484279 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated135942071958528 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 % _timestamp1.7857987974439344e+09 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed139185880116156 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 _runtime3510.270661549 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 135905242 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7139276137472 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 135914900 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated31104697 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed142572939152896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6950985748992 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed142572939152896 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated52484279 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6951073178624 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated53736975 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed132251906005472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed83420965 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed52484277 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 6933974110684 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated142573043621888 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 6951073178624 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current243 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed22632035 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121S2- _wandbruntime3510  _runtime3510:&M OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated53743619 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 135922408 % _timestamp1.7857987978243315e+09 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 135922408 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6951865368576 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated142591090011648 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed83431488 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated22634985 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated135959279075328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 _runtime3510.647670447 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated135639137206272 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed132268646177792 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 trainer/global_step7899 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6951865368576 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7140178072064 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated31108634 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated135639137206272 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current259 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed52490920 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed139203497761698 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6951952805376 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed31108632 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 _step630 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed53743360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed22634728 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed142590985535488 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current257 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6934851583906 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed135639120166912 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed135959254171648 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6934937387142 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated142591090011648 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 6951952805376 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated83441158 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 135932080 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed135639120166912 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated83441158 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated52490922 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed53743360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed22634728 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed142590985535488 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current257 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 6934851583906 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed135639120166912 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed135959254171648 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6934937387142 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated142591090011648 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 6951952805376 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated83441158 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 135932080 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ;OwWCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed135639120166912 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated83441158 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated52490922 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed132268646177792 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 trainer/global_step7899 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6951865368576 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7140178072064 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated31108634 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated135639137206272 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current259 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed52490920 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed139203497761698 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6951952805376 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed31108632 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 _step630$2- _wandbruntime3510  _runtime3510V _runtime 3510.6489159 /loss/FRt2/detection_loss2.4573294117494866 0loss/SRqqt1/detection_loss2.171060976229216 0loss/SRqqt2/detection_loss1.064214519687466 %loss/total_loss18.28062756733427 epoch0 /loss/FRt1/assignment_loss2.897021161749008 2loss/SRqqt1/assignment_loss1.1244187172698348 0loss/FBt1/assignment_loss0.0276025224551331 /loss/FBt2/detection_loss1.5877505214884877 _step631 2loss/SRqqt2/assignment_loss1.1928246545677001 /loss/FBt1/detection_loss0.7427618183816472 % _timestamp1.7857987978260632e+09 0loss/FRt2/assignment_loss2.4718751561494523 1loss/FBt2/assignment_loss0.02373515992198305 .loss/FRt1/detection_loss2.520032947684856 trainer/global_step7899߻4 0loss/FRt2/assignment_loss2.4718751561494523 1loss/FBt2/assignment_loss0.02373515992198305 .loss/FRt1/detection_loss2.520032947684856 trainer/global_step7899 _runtime 3510.6489159 /loss/FRt2/detection_loss2.4573294117494866 0loss/SRqqt1/detection_loss2.171060976229216 0loss/SRqqt2/detection_loss1.064214519687466 %loss/total_loss18.28062756733427 epoch0 /loss/FRt1/assignment_loss2.897021161749008 2loss/SRqqt1/assignment_loss1.1244187172698348 0loss/FBt1/assignment_loss0.0276025224551331 /loss/FBt2/detection_loss1.5877505214884877 _step631 2loss/SRqqt2/assignment_loss1.1928246545677001 /loss/FBt1/detection_loss0.7427618183816472 % _timestamp1.7857987978260632e+0978W(: ʹU gpu.0.gpu41.0 gpu.0.memory3, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts123.039' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent41.013 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0V(:  ֈV disk.drbd1938.out0 disk.drbd1931.in0 disk.rbd1.out 13.951171875 disk.drbd1550.in0 disk.sda1.out0 disk.drbd1673.in0 disk.drbd1289.out0 disk.drbd1285.out0 disk.drbd1289.in0 disk.sda.in0 disk.drbd2118.out0 disk.drbd2111.in0$ disk./.usageGB578.3438949584961! disk.dm-0.out165.24951171875 disk.sda2.out 95.984375 disk.drbd1817.out0 disk.drbd1938.in0 disk.drbd1510.in0 disk.dm-0.in 2.47265625 disk.drbd2078.in0 disk.drbd2111.out0$ memory_percent41.23774170875549* proc.memory.percent17.182016372680664 proc.cpu.threads29 disk.drbd1584.in0 disk.drbd1584.out0 disk.drbd1356.in0! proc.memory.rssMB 11260.40625 disk.drbd1828.out0 disk.drbd1817.in0 disk.drbd2041.in0 network.recv465525 disk.drbd1550.out0 disk.drbd1285.in0 disk.drbd1673.out0 network.sent 7.225063e+06 disk.rbd0.in0 disk.drbd2118.in0 disk.sda2.in0 disk.drbd1552.in0* proc.memory.availableMB38510.43359375 disk.drbd1226.in0 disk.drbd2078.out0 disk.drbd2041.out0* disk./.usagePercent32.344018208333516 disk.drbd1828.in0 disk.sda1.in0 disk.nvme0n1.in 2.47265625 disk.drbd1965.in0 disk.drbd1965.out0 cpu13.827286244154731 disk.drbd1510.out0$ disk.nvme0n1.out165.24951171875 disk.drbd1356.out0 disk.drbd1931.out0 disk.drbd1552.out0 disk.rbd0.out0 disk.sda.out 95.984375 disk.drbd1226.out0 disk.rbd1.in0"?2- _wandbruntime3531  _runtime3531& A trainer/global_step7949 % _timestamp1.7857988189846668e+09 _runtime3531.809715878 _step632 ' lr-AdamW/pg10.00020486311037169817 ' lr-AdamW/pg20.00020486311037169817$| _step632 ' lr-AdamW/pg10.00020486311037169817 ' lr-AdamW/pg20.00020486311037169817 trainer/global_step7949 % _timestamp1.7857988189846668e+09 _runtime3531.809715878nA2- _wandbruntime3531  _runtime3531۪A'O G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed54075875 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated52816429 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed143475273438720 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 6995069314048 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 136775052 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current248 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated136802427798528 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed136802402894848 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 136765394 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed31301545 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed140066777195664 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 % _timestamp1.7857988189853454e+09 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated143475377907712 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed136480291554304 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed22774330 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated52816429 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 6994981884416 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6995069314048 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6994981884416 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated83958623 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed143475273438720 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated136480308593664 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 7184325541376 _step633 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated143986881916928 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7184454118400 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated136480308593664 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated31301547 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated143475377907712 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6977948377316 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated54076125 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed83948967 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0H'O WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed83948967 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed136480291554304 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated83958623 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated83958623 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed143475273438720 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated136480308593664 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 7184325541376 _step633 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated143986881916928 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7184454118400 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated136480308593664 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated31301547 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated143475377907712 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 6977948377316 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated54076125 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed83948967 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed54075875 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated52816429 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed143475273438720 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 6995069314048 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 136775052 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current248 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated136802427798528 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed136802402894848 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 136765394 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed31301545 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed140066777195664 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 % _timestamp1.7857988189853454e+09 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated143475377907712 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed136480291554304 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed22774330 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated52816429 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 6994981884416 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 6995069314048 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 6994981884416 J6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated22777468 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 7185221691392 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 =7DeviceStatsMo6G#nitor.on_train_batch_end/num_alloc_retries0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 % _timestamp1.7857988193562176e+09 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed133105654793792 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated136497475356672 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 _step634 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated143493424297472 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated143493424297472 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6995861504000 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7185360747008 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated133105671833152 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6995948940800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6995861504000 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated136497475356672 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated136819634915328 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated83969160 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed140084394841206 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 136782560 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated52823072 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 163959296 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 _runtime3532.179683358 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 6995948940800 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 136792232 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated144004995662336 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current261 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current263 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed54082689 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated31305484 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed52823070 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed136497458317312 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated54082952 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed22777207 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 136792232 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed31305482 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed144004831703040 trainer/global_step7949 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed83959490 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360D&M REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated83969160 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed140084394841206 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 136782560 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated52823072 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 163959296 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 _runtime3532.179683358 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 6995948940800 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 136792232 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated144004995662336 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current261 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current263 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed54082689 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated31305484 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed52823070 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed136497458317312 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated54082952 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed22777207 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 136792232 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed31305482 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed144004831703040 trainer/global_step7949 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed83959490 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed136819610011648 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 6978825850650 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated22777468 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 7185221691392 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 % _timestamp1.7857988193562176e+09 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed133105654793792 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated136497475356672 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 _step634 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated143493424297472 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated143493424297472 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 6995861504000 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7185360747008 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated133105671833152 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 6995948940800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 6995861504000 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated136497475356672 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated136819634915328RYa2- _wandbruntime3532  _runtime3532+ 2loss/SRqqt1/assignment_loss0.9108782834373415 3loss/FBt1/assignment_loss0.0034981216737196594 /loss/FRt2/detection_loss2.4324284967035057 1loss/SRqqt1/detection_loss1.8037084996700288 epoch0 _runtime3532.181112339 -loss/FRt1/detection_loss2.47737462938517 % _timestamp1.7857988193581796e+09 0loss/FRt1/assignment_loss2.9805823242527314 1loss/SRqqt2/detection_loss1.0868581522256135 &loss/total_loss17.582952888837404 0loss/FRt2/assignment_loss2.2686777830123903 1loss/SRqqt2/assignment_loss1.003401769232005 1loss/FBt2/assignment_loss0.10118298747362608 /loss/FBt1/detection_loss0.6801485877293043 /loss/FBt2/detection_loss1.8342132540419698 trainer/global_step7949 _step635R>  0loss/FRt1/assignment_loss2.9805823242527314 1loss/SRqqt2/detection_loss1.0868581522256135 &loss/total_loss17.582952888837404 0loss/FRt2/assignment_loss2.2686777830123903 1loss/SRqqt2/assignment_loss1.003401769232005 1loss/FBt2/assignment_loss0.10118298747362608 /loss/FBt1/detection_loss0.6801485877293043 /loss/FBt2/detection_loss1.8342132540419698 trainer/global_step7949 _step635 2loss/SRqqt1/assignment_loss0.9108782834373415 3loss/FBt1/assignment_loss0.0034981216737196594 /loss/FRt2/detection_loss2.4324284967035057 1loss/SRqqt1/detection_loss1.8037084996700288 epoch0 _runtime3532.181112339 -loss/FRt1/detection_loss2.47737462938517 % _timestamp1.7857988193581796e+09(: U gpu.0.gpu24.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts119.218' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent39.739333333333335 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated137662783638528 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed53148577 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated84486625 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated144892364285952 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 7038978019840 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated22918582 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed84476969 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated144377712193536 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 trainer/global_step7999 % _timestamp1.7857988412215292e+09 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated144377712193536 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed137338629704704 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated53148579 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed133925923237472 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated54416979 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed144377607724544 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 _runtime3554.045827785 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7229580647424 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated53148579 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 _step637 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated137338646744064 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 7039065449472 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed84476969 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed31498395 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 7229443681792 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7039065449472 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current258 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated140947777117656 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed137662758734848 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated133925940276832 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak1 0.1887 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 137625546 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed54416719 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed140947674275172 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0LӍ'O PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated53148579 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 _step637 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated137338646744064 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 7039065449472 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed84476969 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed31498395 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 7229443681792 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7039065449472 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current258 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated140947777117656 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed137662758734848 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated133925940276832 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 137625546 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed54416719 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed140947674275172 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated31498397 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 7021836840824 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated84486625 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed137338629704704 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 137635204 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated137662783638528 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed53148577 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated84486625 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated144892364285952 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 7038978019840 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated22918582 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed84476969 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated144377712193536 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 trainer/global_step7999 % _timestamp1.7857988412215292e+09 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated144377712193536 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed137338629704704 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated53148579 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed133925923237472 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated54416979 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed144377607724544 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 _runtime3554.045827785 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7229580647424 60DeviceStatsMonitor.on_train_batch_start/num_ooms01G2- _wandbruntime3554  _runtime3554kd&M J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed84487492 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 7230351726592 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated31502334 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 trainer/global_step7999 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated144910471051776 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated140965394763310 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated53155222 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed137679965851648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 _step638 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated54423754 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 % _timestamp1.7857988416054566e+09 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 137642712 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 7039857639424 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated22921420 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7230480296448 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated137355813507072 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated84497162 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed31502332 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed137355796467712 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 7039945076224 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 137652384 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed53155220 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed22921171 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed140965291920714 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated144395758583296 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated137679990755328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated84497162 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 7039945076224 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7022628510922 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 7039857639424 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated137355813507072 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 137652384 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _runtime3554.428788369 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed54423503 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed133942663409792Ѱz&M _step638 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated54423754 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 % _timestamp1.7857988416054566e+09 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 137642712 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 7039857639424 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated22921420 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7230480296448 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated137355813507072 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated84497162 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed31502332 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed137355796467712 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 7039945076224 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 137652384 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed53155220 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed22921171 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed140965291920714 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated144395758583296 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated137679990755328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated84497162 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 7039945076224 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7022628510922 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 7039857639424 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated137355813507072 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 137652384 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _runtime3554.428788369 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed54423503 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed133942663409792 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed84487492 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 7230351726592 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated31502334 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 trainer/global_step7999 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated144910471051776 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated140965394763310 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated53155222 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed137679965851648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 27682406402- _wandbruntime3554  _runtime3554  3loss/FBt2/assignment_loss6.973624003592249e-06 /loss/FBt1/detection_loss1.0854084817125487 _runtime3554.429888516 2loss/SRqqt2/assignment_loss0.9463473306111011 /loss/FRt2/detection_loss2.2527820358470994 /loss/FBt2/detection_loss1.4109263002173975 % _timestamp1.7857988416070673e+09 /loss/FRt2/assignment_loss2.259377681479162 1loss/SRqqt1/assignment_loss1.167797379506131 3loss/FBt1/assignment_loss2.991942829784951e-05 1loss/SRqqt1/detection_loss1.9938139875108998 &loss/total_loss17.767686952107844 epoch0 trainer/global_step7999 _step639 /loss/FRt1/assignment_loss3.194308151927175 /loss/FRt1/detection_loss2.5561251942115493 1loss/SRqqt2/detection_loss0.9007635160324801+D 2loss/SRqqt2/assignment_loss0.9463473306111011 /loss/FRt2/detection_loss2.2527820358470994 /loss/FBt2/detection_loss1.4109263002173975 % _timestamp1.7857988416070673e+09 /loss/FRt2/assignment_loss2.259377681479162 1loss/SRqqt1/assignment_loss1.167797379506131 3loss/FBt1/assignment_loss2.991942829784951e-05 1loss/SRqqt1/detection_loss1.9938139875108998 &loss/total_loss17.767686952107844 epoch0 trainer/global_step7999 _step639 /loss/FRt1/assignment_loss3.194308151927175 /loss/FRt1/detection_loss2.5561251942115493 1loss/SRqqt2/detection_loss0.9007635160324801 3loss/FBt2/assignment_loss6.973624003592249e-06 /loss/FBt1/detection_loss1.0854084817125487 _runtime3554.429888516g(: U gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocahOstedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts100.231' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent33.410333333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0m[(:  V$ disk./.usageGB578.3445053100586 disk.drbd1817.out0 disk.drbd2041.out0* proc.memory.availableMB38509.30859375 disk.rbd0.out0 disk.sda.out 96.91015625 disk.drbd2111.in0 disk.drbd1552.out0 proc.cpu.threads29 disk.drbd2118.in0 disk.drbd2118.out0 disk.drbd1356.in0 disk.sda2.in0 disk.drbd1965.in0 disk.drbd1552.in0 disk.drbd1289.in0 disk.nvme0n1.in 2.47265625 disk.drbd1965.out0 disk.drbd1931.out0 disk.dm-0.in 2.47265625 disk.drbd1673.in0 disk.drbd1828.in0* proc.memory.percent17.182016372680664 disk.drbd1550.in0 disk.sda.in0 disk.drbd1510.out0 network.recv467385 disk.drbd2078.out0 disk.drbd1938.out0 disk.drbd1584.out0 disk.rbd1.in0 disk.drbd1828.out0% memory_percent41.239458322525024 disk.drbd1510.in0 disk.drbd1289.out0 disk.drbd1584.in0 disk.drbd1817.in0 disk.drbd1931.in0) disk./.usagePercent32.34405234238805$ disk.nvme0n1.out165.75537109375! proc.memory.rssMB 11260.40625 disk.rbd0.in0 disk.sda1.in0 disk.drbd1285.out0 disk.drbd2078.in0 disk.drbd1673.out0 disk.drbd1550.out0 disk.drbd1226.in0 disk.drbd1226.out0 disk.drbd2111.out0 disk.rbd1.out 14.0771484375 disk.drbd1938.in0 disk.sda1.out0 cpu13.824920691255725 network.sent 7.270254e+06 disk.drbd1285.in0! disk.dm-0.out165.75537109375 disk.drbd1356.out0 disk.sda2.out 96.91015625 disk.drbd2041.in0O(: ƆU gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts125.854' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent41.95133333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors09U(:  ЖU$ disk./.usageGB578.3444862365723 disk.drbd1584.in0 disk.drbd1817.out0 disk.dm-0.in 2.47265625 disk.dm-0.out 166.095703125" disk.nvme0n1.out 166.095703125 disk.rbd0.out0 disk.drbd1510.out0 disk.drbd1226.out0 disk.drbd1356.in0 disk.drbd1828.in0 disk.drbd2111.in0 disk.drbd1965.in0 disk.drbd1931.out0 disk.drbd1552.out0 disk.drbd1550.out0 network.sent 7.270362e+06 disk.drbd1584.out0 disk.rbd1.out 14.1396484375 disk.drbd1552.in0 disk.drbd2118.out0 disk.sda2.out 97.49609375 disk.drbd1673.in0 disk.drbd1938.out0$ memory_percent41.23880863189697 proc.cpu.threads29 network.recv467493 disk.drbd2078.out0! proc.memory.rssMB 11260.40625 disk.sda.in0 disk.drbd2118.in0 cpu13.823686582008389 disk.sda1.out0 disk.drbd1285.in0 disk.sda2.in0 disk.drbd1817.in0 disk.rbd1.in0 disk.drbd2041.in0 disk.rbd0.in0 disk.drbd2111.out0 disk.drbd1285.out0 disk.drbd1510.in0 disk.drbd1226.in0( proc.memory.availableMB 38509.734375* proc.memory.percent17.182016372680664* disk./.usagePercent32.344051275698845 disk.sda.out 97.49609375 disk.drbd2078.in0 disk.nvme0n1.in 2.47265625 disk.drbd1289.out0 disk.sda1.in0 disk.drbd1673.out0 disk.drbd1828.out0 disk.drbd1550.in0 disk.drbd1938.in0 disk.drbd1931.in0 disk.drbd1356.out0 disk.drbd1965.out0 disk.drbd2041.out0 disk.drbd1289.in0LX2- _wandbruntime3576  _runtime357611 $ _timestamp1.785798863231466e+09 _runtime 3576.05445575 _step640 ' lr-AdamW/pg10.00020744032902022877 ' lr-AdamW/pg20.00020744032902022877 trainer/global_step8049 1 $ _timestamp1.785798863231466e+09 _runtime 3576.05445575 _step640 ' lr-AdamW/pg10.00020744032902022877 ' lr-AdamW/pg20.00020744032902022877 trainer/global_step8049g2- _wandbruntime3576  _runtime3576K'O L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed85004971 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7083061584896 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 7082974155264 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed145279942010368 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated23061386 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 _runtime3576.056128678 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed31695245 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated145280046479360 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated138523139478528 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated145280046479360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed138196967855104 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed138523114574848 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 _step641 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current260 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed85004971 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 trainer/global_step8049 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 % _timestamp1.7857988632325287e+09 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed53480727 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed138196967855104 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed23061126 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed145797734465536 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed141828571354680 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 7065725304332 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0;'O TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed31695245 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated145280046479360 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated138523139478528 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated145280046479360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed138196967855104 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed138523114574848 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 _step641 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current260 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed85004971 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 trainer/global_step8049 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 % _timestamp1.7857988632325287e+09 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed53480727 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed138196967855104 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed23061126 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed145797734465536 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed141828571354680 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 7065725304332 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed85004971 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7083061584896 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 7082974155264 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed145279942010368 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated23061386 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 _runtime3576.056128678-2- _wandbruntime3576  _runtime3576\DqwM D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed53487370 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 7066602777666 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated138214151657472 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 % _timestamp1.7857988636120021e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated145298092869120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated23064274 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed85015494 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _step642 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated134779689065152 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 7275526919680 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed145815848611328 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed141846189000222 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 138502864 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 7083941211648 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 138502864 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 138512536 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed134779672025792 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated85025164 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed54763200 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated53487372 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 7083853774848 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 138512536 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated54763458 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated145816002084864 L=DeviceStatsMonitor.on_train_batch_end/reserved_by$1Oetes.all.peak 3021996032 _runtime3576.435323959 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed138540321691648 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated85025164 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated53487372 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current256 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated141846291842818 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 trainer/global_step8049 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed138214134618112 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed23064018 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated145298092869120 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed138214134618112 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated138540346595328фi&M S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed145297988392960 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current258 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated31699184 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed53487370 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 7083853774848 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 7275526919680 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed145815848611328 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed141846189000222 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 138502864 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 7083941211648 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 138502864 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 138512536 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed134779672025792 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated85025164 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed54763200 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated53487372 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 7083853774848 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 138512536 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated54763458 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated145816002084864 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 _runtime3576.435323959 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed138540321691648 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated85025164 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated53487372 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current256 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated141846291842818 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 trainer/global_step8049 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed138214134618112 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed23064018 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated145298092869120 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed138214134618112 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated138540346595328 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed53487370 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 7066602777666 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated138214151657472 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 % _timestamp1.7857988636120021e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated145298092869120 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated23064274 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed85015494 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _step642 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated134779689065152 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 Eg(: ݏU gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts128.616' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.87200000000001 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors01qV(:  U disk.drbd1938.in0 disk.sda.out 97.7421875 disk.drbd2078.in0$ memory_percent41.23886227607727 disk.rbd1.out 14.1396484375 disk.drbd1931.out0 disk.drbd1510.out0 disk.sda2.in0 disk.drbd1965.out0 disk.nvme0n1.in 2.47265625# disk.nvme0n1.out166.4150390625 disk.drbd1356.out0 disk.drbd1673.out0 disk.drbd1550.out0 disk.sda1.in0 disk.drbd1285.in0 disk.drbd1226.out0 disk.sda2.out 97.7421875 disk.drbd2111.in0* proc.memory.availableMB38509.69921875 disk.drbd1552.out0 disk.drbd1550.in0 disk.drbd1285.out0 disk.drbd1510.in0 disk.drbd1226.in0 disk.drbd2078.out0 network.sent 7.318104e+06 disk.rbd1.in0 disk.drbd2118.out0 cpu13.822664412211605 proc.cpu.threads29 disk.rbd0.in0 network.recv469904 disk.drbd1965.in0 disk.rbd0.out0 disk.drbd1356.in0! proc.memory.rssMB 11260.40625) disk./.usagePercent32.34405234238805 disk.drbd1817.in0 disk.drbd1289.out0 disk.drbd1584.out0 disk.drbd2111.out0 disk.drbd1673.in0 disk.drbd1828.out0 disk.dm-0.out166.4150390625 disk.drbd1931.in0 disk.drbd1828.in0 disk.drbd2041.in0 disk.drbd2041.out0 disk.drbd1938.out0 disk.dm-0.in 2.47265625* proc.memory.percent17.182016372680664$ disk./.usageGB578.3445053100586 disk.drbd1817.out0 disk.drbd1552.in0 disk.drbd1289.in0 disk.sda1.out0 disk.sda.in0 disk.drbd1584.in0 disk.drbd2118.in0F 2- _wandbruntime3598  _runtime3598rX ' lr-AdamW/pg10.00020872893834449408 ' lr-AdamW/pg20.00020872893834449408 trainer/global_step8099 % _timestamp1.7857988851832228e+09 _runtime3598.006164724 _step644 # ' lr-AdamW/pg20.00020872893834449408 trainer/global_step8099 % _timestamp1.7857988851832228e+09 _runtime3598.006164724 _step644 ' lr-AdamW/pg10.00020872893834449408Kߡ2- _wandbruntime3598  _runtime3598Ux'O \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 139345850 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated146182380765184 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current273 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated139055323044864 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated53812879 _runtime3598.007405841 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 7319755176960 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed139055306005504 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 7126970290688 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed53812877 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed55096105 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed85532973 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated139383495318528 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated146182380765184 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 7127057720320 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed135599940469472 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated85542629 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed23204010 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 _step645 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated55096380 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 7109527964716 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7319894239744 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated139055323044864 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed85532973 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 % _timestamp1.7857988851841636e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed53812877 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed146182276296192 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated146703389558272 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated142709571276672 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated53812879 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed146182276296192 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated139055323044864 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed85532973 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 % _timestamp1.7857988851841636e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed53812877 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed146182276296192 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated146703389558272 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated142709571276672 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated53812879 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed146182276296192 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed85532973 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated139383495318528 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated146182380765184 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 7127057720320 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed135599940469472 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated85542629 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed23204010 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 _step645 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated55096380 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 7109527964716 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7319894239744 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated146721497077760 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed146721331021312 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated85553166 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed31896032 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 141152768 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated135616697681152 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 _step646 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated53819522 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed23206883 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 _runtime3598.384059806 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated142727188922326 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated23207131 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed142727086079730 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 139372688 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated55103165 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 7320653489664 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated139072489807872 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated139072489807872 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 139363016 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed139400677531648 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated139400702435328 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated146200427154944 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated53819522 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed85543496 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 166056448 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated146200427154944 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed55102915 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 % _timestamp1.7857988855607278e+09 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 139372688 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 trainer/global_step8099 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed85543496 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed146200322678784\A&M J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed53819520 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 7127849910272 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7110405437938 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated146721497077760 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed146721331021312 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated85553166 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed31896032 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 141152768 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated135616697681152 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 _step646 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated53819522 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed23206883 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 _runtime3598.384059806 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated142727188922326 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated23207131 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed142727086079730 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 139372688 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated55103165 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 7320653489664 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated139072489807872 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated139072489807872 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 139363016 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed139400677531648 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated139400702435328 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated146200427154944 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated53819522 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed85543496 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 166056448 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated146200427154944 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed55102915 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 % _timestamp1.7857988855607278e+09 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 139372688 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 trainer/global_step8099 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed85543496 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed146200322678784 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7320794642432 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 FDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed147084610582016 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated147084715051008 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 7171053855744 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated32088947 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated143590468356180 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated54145029 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed86060975 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 $ _timestamp1.785798907674939e+09 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 140206002 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed54145027 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed32088945 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 7153416428224 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed54145027 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed55434627 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated86070631 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated54145029 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0^'O L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed54145027 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed32088945 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 7153416428224 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed54145027 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed55434627 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated86070631 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated54145029 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 7364897380864 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed147084610582016 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated147084715051008 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 7171053855744 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated32088947 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated143590468356180 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated54145029 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed86060975 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 $ _timestamp1.785798907674939e+09 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 140206002 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak2512- _wandbruntime3620  _runtime3620tS;&M SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated23348775 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed147626834072576 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed55441411 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated147626993837568 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 140223168 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated32092884 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed136453689257792 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed54151670 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 7171933482496 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 % _timestamp1.7857989080453448e+09 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 140223168 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed143607983159238 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed32092882 _step650 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated54151672 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed140261033371648 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed147102656964608 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated86081168 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 140232840 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 140232840 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated147102761440768 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current246 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated139930827958272 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed54151670 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated136453706297152 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 7365800700928 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 7171846045696 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7365935562240 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current248 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated147102761440768 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7154293901446 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated140261058275328 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 7171846045696 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated143608086001834 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 trainer/global_step8149 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated55441659 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 7154379704682 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated54151672 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current246 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated139930827958272 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed54151670 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated136453706297152 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 7365800700928 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 7171846045696 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7365935562240 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current248 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated147102761440768 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7154293901446 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated140261058275328 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 7171846045696 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated143608086001834 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 trainer/global_step8149 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated55441659 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 7154379704682 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated54151672 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed147626834072576 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed55441411 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated147626993837568 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 D6DeviceStatDsMonitor.on_train_batch_end/active.all.freed 140223168 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated32092884 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed136453689257792 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed54151670 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 7171933482496 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 % _timestamp1.7857989080453448e+09 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 140223168 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed143607983159238 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed32092882 _step650 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated54151672 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed140261033371648 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed147102656964608 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated86081168 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 140232840 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 140232840 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392)I2- _wandbruntime3620  _runtime36205v{  0loss/FRt1/assignment_loss3.2104471896005715 3loss/FBt2/assignment_loss0.0016121747902633704 /loss/FRt2/detection_loss2.2786921924729455 &loss/total_loss18.256853253886078 0loss/FRt2/assignment_loss2.4604840593776482 2loss/SRqqt1/assignment_loss1.1231818437880399 2loss/SRqqt2/assignment_loss0.8797971270866347 1loss/SRqqt1/detection_loss1.8070320885400384 $ _timestamp1.785798908046906e+09 _runtime3620.870374057 _step651 /loss/FRt1/detection_loss2.4428121149539948 1loss/SRqqt2/detection_loss1.0279802343325206 epoch0 trainer/global_step8149 3loss/FBt1/assignment_loss0.0006234879058878206 /loss/FBt1/detection_loss1.0102576900739222 /loss/FBt2/detection_loss2.0139330509636135l!I 3loss/FBt1/assignment_loss0.0006234879058878206 /loss/FBt1/detection_loss1.0102576900739222 /loss/FBt2/detection_loss2.0139330509636135 0loss/FRt1/assignment_loss3.2104471896005715 3loss/FBt2/assignment_loss0.0016121747902633704 /loss/FRt2/detection_loss2.2786921924729455 &loss/total_loss18.256853253886078 0loss/FRt2/assignment_loss2.4604840593776482 2loss/SRqqt1/assignment_loss1.1231818437880399 2loss/SRqqt2/assignment_loss0.8797971270866347 1loss/SRqqt1/detection_loss1.8070320885400384 $ _timestamp1.785798908046906e+09 _runtime3620.870374057 _step651 /loss/FRt1/detection_loss2.4428121149539948 1loss/SRqqt2/detection_loss1.0279802343325206 epoch0 trainer/global_step8149K(: ְU gpu.0.gpu0.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts95.778' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.926000000000005 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0!VP(:  ǃV disk.drbd1965.out0 disk.drbd1828.out0$ disk./.usageGB578.3445434570312 disk.drbd1817.out0 disk.drbd1673.out0 disk.drbd1931.out0 disk.drbd2078.out0 disk.drbd1965.in0 proc.cpu.threads29 disk.rbd1.in0 disk.sda.in0 disk.nvme0n1.in 2.47265625 disk.drbd2041.in0$ memory_percent41.23954772949219 disk.drbd1584.out0 disk.drbd2118.out0 disk.drbd1356.out0 disk.drbd2111.out0 disk.drbd1285.in0$ proc.memory.availableMB38509.25 disk.drbd1584.in0 disk.drbd1828.in0 disk.drbd1550.in0 disk.drbd1510.out0 disk.drbd1226.in0* proc.memory.percent17.182016372680664 disk.rbd1.out 14.5302734375 disk.drbd1552.in0 disk.drbd1550.out0 disk.sda1.out0 disk.drbd1510.in0 disk.dm-0.in 2.47265625 disk.sda2.in0 disk.rbd0.in0 disk.drbd1285.out0 disk.drbd1226.out0 disk.drbd1673.in0! disk.dm-0.out167.78662109375 disk.drbd1356.in0 disk.sda2.out 99.578125 disk.drbd2041.out0 disk.drbd1289.in0 disk.drbd1938.out0 disk.drbd1289.out0 network.sent 7.366014e+06 network.recv472507 disk.drbd2118.in0 disk.drbd2111.in0 disk.drbd1552.out0 disk.sda.out 99.578125 disk.sda1.in0 disk.drbd1931.in0) disk./.usagePercent32.34405447576646 disk.drbd2078.in0 disk.drbd1938.in0 cpu13.819105736383506$ disk.nvme0n1.out167.78662109375 disk.drbd1817.in0 disk.rbd0.out0! proc.memory.rssMB 11260.40625, D2- _wandbruntime3642  _runtime3642!g  trainer/global_step8199 % _timestamp1.7857989295501013e+09 _runtime3642.373064471 _step652 & lr-AdamW/pg10.0002113061569930247 & lr-AdamW/pg20.0002113061569930247SzH & lr-AdamW/pg10.0002113061569930247 & lr-AdamW/pg20.0002113061569930247 trainer/global_step8199 % _timestamp1.7857989295501013e+09 _runtime3642.373064471 _step6522- _wandbruntime3642  _runtime3642;4K'O VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed86588977 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed54477177 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated147987049336832 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated137273974740832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 141066154 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 7215049991168 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated54477179 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated55774193 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 141075812 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 7197304891732 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 trainer/global_step8199 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed147986944867840 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated141104206998528 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed86588977 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 7214962561536 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 7214962561536 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed23488141 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated148514420199936 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 _runtime3642.374328932 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed55773936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated23488396 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated54477179 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current257 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed148514266719232 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 7197390694856 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed147986944867840 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed32285795 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed141104182094848 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed140771982306304 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 7410084624384 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated86598633 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7410213201408 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed140771982306304 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed137273957701472 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 _step653 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7215049991168 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 141066154 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 7215049991168 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated54477179 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated55774193 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 141075812 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 7197304891732 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 trainer/global_step8199 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed147986944867840 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated141104206998528 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed86588977 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 7214962561536 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 7214962561536 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed23488141 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated148514420199936 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 _runtime3642.374328932 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed55773936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated23488396 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated54477179 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current257 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed148514266719232 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 7197390694856 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed147986944867840 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed32285795 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed141104182094848 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed140771982306304 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 7410084624384 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated86598633 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7410213201408 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed140771982306304 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed137273957701472 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 _step653 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7215049991168 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 141092992 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed148532376416768 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed54483820 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated140789166108672 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated141121414115328 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated148005095726592 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated137290714913152 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed86599500 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 4.DeviceStatsMonitor.on_train_b%atch_end/num_ooms0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 _step654 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated148005095726592 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed140789149069312 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 _runtime3642.743507427 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed32289732 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 141083320 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 7215842181120 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current246 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 7215842181120 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed86599500 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7411115774976 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed137290697873792 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 141092992 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated55780983 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current244 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed141121389211648 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 7198268168190 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed55780737 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated148532529890304 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed144488880238746 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7198182364954 trainer/global_step8199 % _timestamp1.7857989299201574e+09 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated32289734 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed140789149069312''&M KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed55780737 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated148532529890304 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed144488880238746 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7198182364954 trainer/global_step8199 % _timestamp1.7857989299201574e+09 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated32289734 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed140789149069312 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 7215929617920 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 141092992 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed148532376416768 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed54483820 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated140789166108672 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated141121414115328 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated148005095726592 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated137290714913152 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed86599500 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 _step654 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated148005095726592 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed140789149069312 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 _runtime3642.743507427 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed32289732 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 141083320 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 7215842181120 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current246 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 7215842181120 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed86599500 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7411115774976 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed137290697873792 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 141092992 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated55780983 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current244 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed141121389211648 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 7198268168190 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0F:*2- _wandbruntime3642  _runtime3642w 0loss/SRqqt2/assignment_loss1.24058789703995 /loss/FBt1/detection_loss0.9704116783765573 trainer/global_step8199 % _timestamp1.7857989299217746e+09 /loss/FRt1/assignment_loss2.858143775693832 0loss/FBt1/assignment_loss0.0497342367296573 /loss/FRt2/detection_loss2.4279225951916463 _runtime3642.744636149 1loss/FBt2/assignment_loss0.10283798776071507 1loss/SRqqt2/detection_loss0.9923650662781615 /loss/FBt2/detection_loss2.1870238014962524 %loss/total_loss19.30047785384631 /loss/FRt2/assignment_loss2.090926618994893 .loss/FRt1/detection_loss2.955569784006765 0loss/SRqqt1/detection_loss2.041269310363909 epoch0 _step655 2loss/SRqqt1/assignment_loss1.3836851019139698D 2loss/SRqqt1/assignment_loss1.3836851019139698 0loss/SRqqt2/assignment_loss1.24058789703995 /loss/FBt1/detection_loss0.9704116783765573 trainer/global_step8199 % _timestamp1.7857989299217746e+09 /loss/FRt1/assignment_loss2.858143775693832 0loss/FBt1/assignment_loss0.0497342367296573 /loss/FRt2/detection_loss2.4279225951916463 _runtime3642.744636149 1loss/FBt2/assignment_loss0.10283798776071507 1loss/SRqqt2/detection_loss0.9923650662781615 /loss/FBt2/detection_loss2.1870238014962524 %loss/total_loss19.30047785384631 /loss/FRt2/assignment_loss2.090926618994893 .loss/FRt1/detection_loss2.955569784006765 0loss/SRqqt1/detection_loss2.041269310363909 epoch0 _step655g\(: U gpu.0.gpu28.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts127.881' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent42.626999999999995 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0*áW(:  U disk.drbd1356.in0 disk.drbd2041.out0 disk.drbd1552.out0 disk.drbd1828.out0 network.recv474658 disk.dm-0.in 2.47265625* proc.memory.percent17.182016372680664 disk.drbd2118.out0 disk.drbd1673.in0 disk.drbd1673.out0 disk.drbd1550.in0 disk.sda1.out0 disk.drbd2111.out0 disk.drbd1817.in0 disk.drbd1289.in0$ disk./.usageGB578.3445434570312 disk.drbd1510.in0 disk.drbd1931.out0 disk.drbd1938.out0% memory_percent41.241103410720825 disk.drbd1828.in0 disk.sda1.in0 disk.drbd1584.out0$ disk.nvme0n1.out167.95068359375 disk.drbd1356.out0 network.sent 7.367445e+06 disk.drbd1285.out0 disk.drbd2118.in0 disk.rbd1.out 14.5302734375 disk.drbd1965.in0 disk.drbd1931.in0 disk.sda2.in0 disk.drbd1817.out0 disk.drbd1552.in0 disk.nvme0n1.in 2.47265625! proc.memory.rssMB 11260.40625 cpu13.818188612396265 disk.rbd0.out0 disk.drbd2111.in0 disk.drbd2041.in0 disk.drbd1226.out0 disk.sda2.out 99.859375 disk.rbd1.in0 disk.drbd1938.in0) disk./.usagePercent32.34405447576646 disk.drbd2078.out0 disk.drbd1584.in0 disk.drbd1965.out0 disk.drbd1550.out0 disk.drbd1289.out0 proc.cpu.threads29 disk.rbd0.in0 disk.sda.in0! disk.dm-0.out167.95068359375 disk.drbd1226.in0* proc.memory.availableMB38508.23046875 disk.drbd1510.out0 disk.sda.out 99.859375 disk.drbd1285.in0 disk.drbd2078.in0((: W gpu.0.gpu38.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts120.924' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent40.30800000000001 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors02[(:  W$ memory_percent41.23923182487488 disk.sda1.in0 disk.nvme0n1.in 2.47265625 disk.drbd1552.in0 disk.drbd1828.out0 cpu13.81709196435054 disk.drbd1510.in0 disk.drbd1510.out0 disk.drbd1673.in0 disk.drbd1828.in0 disk.drbd1289.in0 proc.cpu.threads29) disk./.usagePercent32.34404956899612 disk.drbd1226.in0 network.sent 7.510091e+06 disk.rbd0.out0 disk.dm-0.in 2.47265625! disk.dm-0.out168.38623046875 disk.drbd2078.in0 disk.drbd1584.in0 disk.drbd1584.out0 disk.drbd1673.out0 disk.drbd1550.in0 disk.drbd1356.in0 disk.sda2.out 100.27734375 disk.sda1.out0$ disk.nvme0n1.out168.38623046875 disk.drbd1931.out0 disk.drbd1550.out0 disk.drbd1938.out0 disk.sda.in0 disk.drbd1285.in0 disk.drbd1965.in0 disk.drbd2041.out0 disk.drbd1289.out0 disk.drbd1938.in0 disk.drbd2111.in0 disk.drbd1931.in0 disk.sda.out 100.27734375 disk.drbd1817.in0 disk.rbd1.in0 disk.drbd2118.out0 disk.drbd1552.out0 disk.drbd2118.in0 disk.rbd1.out 14.6865234375! proc.memory.rssMB 11260.40625$ disk./.usageGB578.3444557189941 disk.rbd0.in0 disk.drbd2078.out0 disk.sda2.in0 disk.drbd1817.out0* proc.memory.percent17.182016372680664 disk.drbd1285.out0 disk.drbd1226.out0 disk.drbd2041.in0* proc.memory.availableMB38509.45703125 network.recv484310 disk.drbd1356.out0 disk.drbd2111.out0 disk.drbd1965.out0NI2- _wandbruntime3664  _runtime3664 % _timestamp1.7857989515973184e+09 _runtime3664.420290652 _step656 ' lr-AdamW/pg10.00021259476631728998 ' lr-AdamW/pg20.00021259476631728998 trainer/global_step8249s  ' lr-AdamW/pg20.00021259476631728998 trainer/global_step8249 % _timestamp1.7857989515973184e+09 _runtime3664.420290652 _step656 ' lr-AdamW/pg10.00021259476631728998=2- _wandbruntime3664  _runtime3664ˉn%O LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current247 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 141926306 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated149419946825216 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated56113802 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current245 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated23631155 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed87116979 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated54809329 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7455383986688 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed148889279153664 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed32482645 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed54809327 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed56113555 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed145352159672712 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated138110983356832 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 7259046126592 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed148889279153664 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 7241279158364 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated141964562838528 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated145352262515196 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 7258958696960 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 _step657 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 YLDeviceStatsMonitor.on_train_batch_start/Kwinactive_split_bytes.large_pool.peak86278144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7259046126592 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated54809329 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed23630910 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0'O ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 7258958696960 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 _step657 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7259046126592 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated54809329 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed23630910 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current247 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 141926306 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated149419946825216 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated56113802 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current245 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated23631155 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed87116979 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated54809329 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7455383986688 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed148889279153664 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed32482645 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed54809327 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed56113555 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed145352159672712 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated138110983356832 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 7259046126592 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed148889279153664 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 7241279158364 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated141964562838528 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated145352262515196 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0c{w2- _wandbruntime3664  _runtime3664h&M F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current252 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated141981769955328 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated148907430012416 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed141647487219712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 141953144 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed54815970 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed56120290 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed32486582 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 7259925753344 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated141647504259072 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated54815972 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed54815970 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 % _timestamp1.7857989519679935e+09 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed148907325536256 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed87127502 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 141953144 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed145369777318254 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 7259925753344 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7242070828462 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 141943472 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed138127706489792 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated56120544 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated149438049980416&M F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current252 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated141981769955328 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated148907430012416 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed141647487219712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 141953144 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed54815970 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed56120290 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed32486582 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 7259925753344 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated141647504259072 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated54815972 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed54815970 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 % _timestamp1.7857989519679935e+09 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed148907325536256 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed87127502 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 141953144 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed145369777318254 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 7259925753344 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7242070828462 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 141943472 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed138127706489792 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated56120544 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated149438049980416$7 2- _wandbruntime3664  _runtime3664fe /loss/FBt1/detection_loss0.9858914291869629 2loss/SRqqt1/assignment_loss0.9978108321744446 &loss/total_loss17.528613107590026 _runtime3664.792531145 1loss/SRqqt1/detection_loss1.7437882275270247 1loss/SRqqt2/detection_loss1.0031699167487964 % _timestamp1.7857989519696827e+09 0loss/FRt1/assignment_loss2.6848845444619656 2loss/SRqqt2/assignment_loss0.9505588964540002 epoch0 trainer/global_step8249 _step659 /loss/FBt2/detection_loss2.0479363614693282 /loss/FRt2/assignment_loss2.263621843996502 3loss/FBt1/assignment_loss0.0001145883837234946 3loss/FBt2/assignment_loss3.690508819857996e-05 -loss/FRt1/detection_loss2.44452052594473 -loss/FRt2/detection_loss2.40627903615435  1loss/SRqqt1/detection_loss1.7437882275270247 1loss/SRqqt2/detection_loss1.0031699167487964 % _timestamp1.7857989519696827e+09 0loss/FRt1/assignment_loss2.6848845444619656 2loss/SRqqt2/assignment_loss0.9505588964540002 epoch0 trainer/global_step8249 _step659 /loss/FBt2/detection_loss2.0479363614693282 /loss/FRt2/assignment_loss2.263621843996502 3loss/FBt1/assignment_loss0.0001145883837234946 3loss/FBt2/assignment_loss3.690508819857996e-05 -loss/FRt1/detection_loss2.44452052594473 -loss/FRt2/detection_loss2.40627903615435 /loss/FBt1/detection_loss0.9858914291869629 2loss/SRqqt1/assignment_loss0.9978108321744446 &loss/total_loss17.528613107590026 _runtime3664.792531145\k(:  ЇV disk.drbd1965.in0 proc.cpu.threads29 disk.drbd1584.out0 disk.nvme0n1.in 2.47265625 disk.sda2.in0! proc.memory.rssMB 11260.40625 disk.drbd1289.out0 disk.drbd1285.out0 disk.drbd1226.in0# memory_percent41.2392795085907* proc.memory.availableMB38509.42578125$ disk./.usageGB578.3443984985352 disk.drbd1931.in0* proc.memory.percent17.182016372680664 disk.drbd1510.out0 disk.drbd1226.out0 disk.drbd2111.out0 disk.drbd1938.out0 disk.drbd2078.in0 disk.drbd2111.in0 disk.drbd1673.out0 cpu13.816002418868935 disk.sda1.out0 disk.drbd1356.in0 network.recv485235 disk.rbd1.out 14.6865234375 disk.drbd1552.out0 network.sent 7.511109e+06 disk.drbd1550.in0 disk.sda1.in0 disk.rbd0.in0 disk.rbd0.out0 disk.sda.in0 disk.drbd1510.in0 disk.drbd1584.in0$ disk.nvme0n1.out168.56982421875 disk.drbd2041.in0 disk.drbd2118.in0 disk.sda2.out 100.6640625 disk.drbd1552.in0 disk.drbd1289.in0 disk.drbd1285.in0 disk.drbd2118.out0 disk.drbd1817.out0 disk.drbd1550.out0 disk.sda.out 100.6640625 disk.dm-0.in 2.47265625 disk.drbd1673.in0 disk.drbd2041.out0! disk.dm-0.out168.56982421875 disk.rbd1.in0) disk./.usagePercent32.34404636892851 disk.drbd1938.in0 disk.drbd1356.out0 disk.drbd1817.in0 disk.drbd1965.out0 disk.drbd1931.out0 disk.drbd1828.in0 disk.drbd1828.out0 disk.drbd2078.out0(: 錹U gpu.0.gpu24.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts130.95' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent43.64999999999999 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0&82- _wandbruntime3686  _runtime3686nuN _step660 ' lr-AdamW/pg10.00021388337564155532 ' lr-AdamW/pg20.00021388337564155532 trainer/global_step8299 % _timestamp1.7857989735463176e+09 _runtime3686.3692424963c ' lr-AdamW/pg20.00021388337564155532 trainer/global_step8299 % _timestamp1.7857989735463176e+09 _runtime3686.369242496 _step660 ' lr-AdamW/pg10.00021388337564155532UՁ2- _wandbruntime3686  _runtime3686'O N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 142796116 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current244 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed149791613439488 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated87654637 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated56453508 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated138947991972832 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 7303042262016 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed87644981 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 _step661 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 7285081818748 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current246 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7303042262016 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated55141479 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed23773767 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed32679495 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated150325436761088 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 7500387408384 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed138947974933472 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 trainer/global_step8299 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated87654637 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed150325281183232 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 _runtime3686.370472492 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 7285167621872 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated142488675646464 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated142824918678528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed142488658607104 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed142824893774848 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 # _timestamp1.78579897354724e+09 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated142488675646464 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed55141477 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated32679497 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated23774011 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed149791613439488 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed146233056752220 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated149791717908480 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 7302954832384 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 142786458 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated146233159594704 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed56453262 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated149791717908480 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed142488658607104 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7500518082560 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed149791613439488 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated87654637 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated56453508 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated138947991972832 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 7303042262016 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed87644981 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 _step661 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 7285081818748 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current246 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7303042262016 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated55141479 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed23773767 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed32679495 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated150325436761088 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 7500387408384 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed138947974933472 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 trainer/global_step8299 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated87654637 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed150325281183232 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 _runtime3686.370472492 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 7285167621872 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated142488675646464 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated142824918678528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed142488658607104 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed142824893774848 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 # _timestamp1.78579897354724e+09 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated142488675646464 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed55141477 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated32679497 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated23774011 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed149791613439488 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed146233056752220 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated149791717908480 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current22- _wandbruntime3686  _runtime3686[&M ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed56460073 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed87655504 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 _runtime3686.741960994 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7501425152512 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed138964715105792 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated87665174 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated142505842409472 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed23776641 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed150343391182848 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 7303834451968 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed55148120 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed32683432 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated146250777240358 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 7286045095206 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated142505842409472 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed146250674397762 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 142813296 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated32683434 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 _step662 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 7501290291200 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated138964732145152 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 142803624 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated55148122 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed142505825370112 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated149809764298240 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current251 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed55148120 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 7303921888768 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated149809764298240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated55148122 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated56460326 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 142813296 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed142842100891648 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed149809659822080 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 trainer/global_step8299 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 7303834451968 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated23776892 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 % _timestamp1.7857989739185917e+09 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed56460073 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed87655504 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 _runtime3686.741960994 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7501425152512 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed138964715105792 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated87665174 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated142505842409472 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed23776641 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed150343391182848 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 7303834451968 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed55148120 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed32683432 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated146250777240358 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 7286045095206 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated142505842409472 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed146250674397762 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 142813296 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated32683434 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 _step662 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 7501290291200 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated138964732145152 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 142803624 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated55148122 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed142505825370112 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated149809764298240 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current251 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed55148120 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 7303921888768 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated149809764298240 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated55148122 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated56460326 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 142813296 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed142842100891648 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596,2- _wandbruntime3686  _runtime3686(C _runtime3686.743268271 .loss/FRt1/detection_loss2.702655833802725 .loss/FRt2/detection_loss2.439326157958447 2loss/FBt1/assignment_loss0.010651803975312093 1loss/SRqqt1/detection_loss1.7675262949644373 1loss/SRqqt2/detection_loss0.9798404968478395 _step663 0loss/FRt1/assignment_loss2.7271591354357567 2loss/SRqqt1/assignment_loss1.2584791735101204 2loss/SRqqt2/assignment_loss1.0395991172503867 /loss/FBt1/detection_loss0.6365893379164239 &loss/total_loss17.292591365151704 % _timestamp1.7857989739203286e+09 /loss/FRt2/assignment_loss2.268575725676138 1loss/FBt2/assignment_loss0.18108318856820915 /loss/FBt2/detection_loss1.2811050992459059 epoch0 trainer/global_step8299[+  2loss/FBt1/assignment_loss0.010651803975312093 1loss/SRqqt1/detection_loss1.7675262949644373 1loss/SRqqt2/detection_loss0.9798404968478395 _step663 0loss/FRt1/assignment_loss2.7271591354357567 2loss/SRqqt1/assignment_loss1.2584791735101204 2loss/SRqqt2/assignment_loss1.0395991172503867 /loss/FBt1/detection_loss0.6365893379164239 &loss/total_loss17.292591365151704 % _timestamp1.7857989739203286e+09 /loss/FRt2/assignment_loss2.268575725676138 1loss/FBt2/assignment_loss0.18108318856820915 /loss/FBt2/detection_loss1.2811050992459059 epoch0 trainer/global_step8299 _runtime3686.743268271 .loss/FRt1/detection_loss2.702655833802725 .loss/FRt2/detection_loss2.439326157958447] v(: U gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts96.052' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent32.01733333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0̭wX(:  V disk.rbd0.out0 disk.rbd1.out 14.7490234375 disk.drbd1285.out0 disk.drbd1226.in0 disk.drbd2118.out0 disk.sda2.out 100.98046875 disk.drbd1817.out0" disk.nvme0n1.out 169.021484375 proc.cpu.threads29 network.sent 7.558128e+06 disk.drbd1673.in0 disk.drbd1931.out0 network.recv486775$ disk./.usageGB578.3444595336914 disk.drbd1817.in0 disk.drbd1673.out0 disk.drbd1938.in0 disk.dm-0.in 2.47265625 disk.dm-0.out 169.021484375 disk.sda2.in0 disk.drbd1931.in0 disk.sda1.out0 disk.drbd1285.in0 disk.drbd2111.in0 disk.drbd1550.out0 disk.drbd1226.out0$ memory_percent41.23902916908264 disk.drbd1965.out0 disk.sda.out 100.98046875 disk.drbd1510.in0 disk.drbd1356.out0* proc.memory.availableMB38509.58984375 cpu13.814939832104539 disk.drbd1552.out0 disk.drbd1828.in0 disk.drbd1550.in0 disk.drbd1289.out0 disk.sda1.in0 disk.drbd2111.out0 disk.drbd1965.in0 disk.drbd1938.out0 disk.sda.in0 disk.nvme0n1.in 2.47265625) disk./.usagePercent32.34404978233396 disk.rbd1.in0 disk.drbd2041.in0 disk.drbd1828.out0 disk.drbd1510.out0! proc.memory.rssMB 11260.40625 disk.drbd1552.in0 disk.drbd1289.in0 disk.drbd2078.in0 disk.drbd1584.out0 disk.drbd2118.in0* proc.memory.percent17.182016372680664 disk.drbd2041.out0 disk.rbd0.in0 disk.drbd2078.out0 disk.drbd1584.in0 disk.drbd1356.in0wW(:  ҮU" disk.nvme0n1.out 169.197265625 disk.drbd1289.in0 disk.dm-0.in 2.47265625 disk.drbd1965.in0 disk.drbd2078.out0 disk.drbd1584.in0 disk.drbd1356.in0 disk.drbd1356.out0 disk.sda2.in0 disk.drbd2111.out0 disk.drbd1817.out0 disk.rbd1.out 14.7490234375 network.recv488350 disk.drbd1673.in0 cpu13.813890058763787 proc.cpu.threads29$ disk./.usageGB578.3444786071777 disk.sda1.out0* proc.memory.availableMB38509.83984375* proc.memory.percent17.182016372680664 disk.drbd1828.out0 disk.sda.in0 disk.rbd1.in0 disk.drbd1285.out0 disk.drbd1510.in0 disk.sda1.in0 disk.drbd2041.out0 disk.drbd1817.in0 disk.drbd1965.out0 disk.drbd1552.out0 disk.rbd0.out0 disk.dm-0.out 169.197265625 disk.drbd2041.in0 disk.drbd1226.out0$ memory_percent41.23864769935608 disk.drbd1550.in0 disk.drbd1285.in0 disk.drbd1673.out0 disk.drbd1552.in0 disk.drbd1289.out0 disk.drbd2078.in0 disk.sda2.out 101.19921875 disk.rbd0.in0 disk.drbd2118.out0 disk.drbd1931.out0 disk.drbd1550.out0 disk.drbd1938.out0 disk.drbd1584.out0 disk.drbd2118.in0 network.sent 7.60314e+06 disk.drbd1938.in0 disk.drbd1226.in0 disk.drbd2111.in0! proc.memory.rssMB 11260.40625) disk./.usagePercent32.34405084902316 disk.drbd1931.in0 disk.drbd1828.in0 disk.sda.out 101.19921875 disk.drbd1510.out0 disk.nvme0n1.in 2.47265625~ Μ(: ĀV gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts129.701' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent43.233666666666664 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0>L2- _wandbruntime3708  _runtime37087[ _step664 & lr-AdamW/pg10.0002151719849658206 & lr-AdamW/pg20.0002151719849658206 trainer/global_step8349 % _timestamp1.7857989957379472e+09 _runtime3708.562701677{l  & lr-AdamW/pg10.0002151719849658206 & lr-AdamW/pg20.0002151719849658206 trainer/global_step8349 % _timestamp1.7857989957379472e+09 _runtime3708.562701677 _step664!2- _wandbruntime3708  _runtime3708ή'O O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated151231009312768 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed151230851637760 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 7346950967808 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated150694052194304 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 % _timestamp1.7857989957386234e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _step665 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed88172983 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed143346996757504 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed143685249614848 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated139785000588832 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed139784983549472 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed23917276 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7347038397440 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated88182639 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed56793621 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated56793894 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed55473627 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated147114056674212 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed55473627 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated55473629 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated143685274518528 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed147113953831728 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 143656268 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated23917547 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 7328970282256 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated32876347 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current271 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 7545602022912 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7545734794240 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed150693947725312 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 7329056085380 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated150694052194304 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated143347013796864 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed88172983 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed150693947725312 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current273 _runtime3708.562793363 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated55473629 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed32876345 vO TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _step665 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed88172983 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed143346996757504 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed143685249614848 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated139785000588832 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed139784983549472 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed23917276 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7347038397440 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated88182639 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed56793621 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated56793894 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed55473627 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated147114056674212 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed55473627 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated55473629 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated143685274518528 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed147113953831728 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 143656268 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated23917547 JDDeviceStatsMonitoXar.on_train_batch_start/oversize_allocations.current0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 7328970282256 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated32876347 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current271 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 7545602022912 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7545734794240 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed150693947725312 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 7329056085380 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated150694052194304 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated143347013796864 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed88172983 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed150693947725312 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current273 _runtime3708.562793363 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated55473629 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed32876345 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated151231009312768 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed151230851637760 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 7346950967808 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated150694052194304 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 % _timestamp1.7857989957386234e+09:#;2- _wandbruntime3708  _runtime3708..e&M H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated56800658 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 143673448 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed143364163520512 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated150712098584064 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated143364180559872 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated88193176 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 7347830587392 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 143673448 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed150711994107904 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed151248966881280 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated143364180559872 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed88183506 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7546634525184 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 143663776 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated151249116160512 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _runtime3708.939195208 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed32880282 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 $ _timestamp1.785798996115833e+09 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated88193176 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current236 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 149279232 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated32880284 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 124375552 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 7329933558714 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated23920374 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 7546510149632 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed139801723721792 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated143702481635328 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed55480270 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed56800422 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7329847755478 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed55480270 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 7347830587392 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed143364163520512 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated55480272 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated147131674319866 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0FN<&M JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 7329933558714 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated23920374 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 7546510149632 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed139801723721792 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated143702481635328 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed55480270 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed56800422 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7329847755478 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed55480270 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 7347830587392 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed143364163520512 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated55480272 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated147131674319866 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated56800658 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 143673448 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed143364163520512 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated150712098584064 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated143364180559872 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated88193176 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 7347830587392 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 143673448 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed150711994107904 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed151248966881280 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated143364180559872 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed88183506 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7546634525184 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 143663776 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated151249116160512 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _runtime3708.939195208 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed32880282 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 $ _timestamp1.785798996115833e+09 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated88193176 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current236 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 149279232 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated32880284 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 124375552 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 PDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed151596282011136 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed144205334907904 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed88700985 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current247 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated140622009204832 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated144205351947264 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed24061070 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated147994953753720 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.alloca4Q5ʪted144205351947264 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated57134512 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated55805779 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed55805777 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated33073197 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated88710641 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed140621992165472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 144506762 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current245 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 7372944548888 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 7390947103232 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed55805777 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed144205334907904 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7391034532864 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed88700985 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7590970700288 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed88700985 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7590970700288 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed151596282011136 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed144205334907904 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed88700985 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current247 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated140622009204832 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated144205351947264 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed24061070 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated147994953753720 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated144205351947264 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated57134512 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated55805779 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed55805777 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated33073197 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated88710641 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed140621992165472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 144506762 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current245 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 7372944548888 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 7390947103232 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed55805777 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed144205334907904 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7391034532864 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 e2- _wandbruntime3730  _runtime3730g&M H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 144523928 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated24064234 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed152154554162688 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed144222501670912 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed57141114 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated144562837475328 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed144562812571648 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated55812422 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed144222501670912 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed55812420 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated144222518710272 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 7391826722816 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed24063982 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated88721178 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 % _timestamp1.7857990177385848e+09 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 144533600 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed33077132 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated55812422 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated148012571399374 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed148012468556778 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 7391914159616 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated152154713927680 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7591876452352 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current252 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 _step670 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated144222518710272 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 7391914159616 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 trainer/global_step8399 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed151614328393728 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed88711508 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 144533600 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 _runtime3730.562012467 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current254 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated55812422 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated148012571399374 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed148012468556778 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 7391914159616 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated152154713927680 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7591876452352 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current252 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 _step670 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated144222518710272 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 7391914159616 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 trainer/global_step8399 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed151614328393728 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed88711508 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 144533600 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 _runtime3730.562012467 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current254 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_$> bytes.large_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed57141114 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated144562837475328 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed144562812571648 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated55812422 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed144222501670912 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed55812420 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated144222518710272 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 7391826722816 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed24063982 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated88721178(2- _wandbruntime3730  _runtime3730>Y 1loss/SRqqt2/detection_loss1.0475568212745594 /loss/FBt1/detection_loss0.8839377480877662 /loss/FBt2/detection_loss1.7646483331918716 &loss/total_loss18.477192508200005 trainer/global_step8399 % _timestamp1.7857990177398362e+09 _runtime3730.562723684 /loss/FRt2/detection_loss2.6224917764916564 _step671 2loss/SRqqt2/assignment_loss1.0050595268972595 1loss/FBt1/assignment_loss0.07758848693962375 epoch0 0loss/FRt1/assignment_loss3.1626935179640605 /loss/FRt2/assignment_loss2.467279035033602 2loss/SRqqt1/assignment_loss0.9655877283844605 4loss/FBt2/assignment_loss0.00023805020206069116 .loss/FRt1/detection_loss2.631271499686125 0loss/SRqqt1/detection_loss1.848839984046963CT  _runtime3730.562723684 /loss/FRt2/detection_loss2.6224917764916564 _step671 2loss/SRqqt2/assignment_loss1.0050595268972595 1loss/FBt1/assignment_loss0.07758848693962375 epoch0 0loss/FRt1/assignment_loss3.1626935179640605 /loss/FRt2/assignment_loss2.467279035033602 2loss/SRqqt1/assignment_loss0.9655877283844605 4loss/FBt2/assignment_loss0.00023805020206069116 .loss/FRt1/detection_loss2.631271499686125 0loss/SRqqt1/detection_loss1.848839984046963 1loss/SRqqt2/detection_loss1.0475568212745594 /loss/FBt1/detection_loss0.8839377480877662 /loss/FBt2/detection_loss1.7646483331918716 &loss/total_loss18.477192508200005 trainer/global_step8399 % _timestamp1.7857990177398362e+09ƴ(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts127.723' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.57433333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0 Q@2T(:  ⃂V disk.drbd1289.in0 disk.drbd1938.in0 disk.sda.in0 disk.nvme0n1.in 2.47265625 cpu13.81193586361515 disk.sda.out 102.12109375 disk.drbd1931.out0 disk.drbd1828.out0 disk.drbd1673.in0 disk.drbd1550.in0 disk.drbd1285.out0! disk.dm-0.out169.76416015625 disk.sda2.out 102.12109375! proc.memory.rssMB 11260.40625 disk.drbd1510.out0$ disk.nvme0n1.out169.76416015625% memory_percent41.239964962005615 disk.sda1.out0 disk.drbd1931.in0 disk.drbd1828.in0 disk.drbd1965.in0 disk.drbd1285.in0 network.recv491037 disk.drbd1552.in0 disk.drbd1356.out0 proc.cpu.threads29 disk.drbd1938.out0 disk.drbd2078.in0 disk.rbd1.in0) disk./.usagePercent32.34404978233396 disk.rbd0.out0 disk.drbd1817.out0* proc.memory.percent17.182016372680664 disk.drbd1552.out0 disk.dm-0.in 2.47265625 disk.drbd1356.in0 disk.drbd2041.out0 disk.rbd0.in0 disk.drbd1226.out0 disk.drbd2118.in0 disk.drbd2041.in0 disk.drbd1510.in0 disk.drbd1584.out0 disk.drbd2111.in0 disk.rbd1.out14.875 disk.drbd1965.out0) proc.memory.availableMB 38508.9765625 disk.drbd2118.out0 disk.sda2.in0 disk.sda1.in0 network.sent 7.651097e+06$ disk./.usageGB578.3444595336914 disk.drbd1289.out0 disk.drbd1673.out0 disk.drbd1550.out0 disk.drbd1226.in0 disk.drbd2078.out0 disk.drbd1584.in0 disk.drbd2111.out0 disk.drbd1817.in0Rw(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts108.111' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent36.037 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0/yT(:  ـU disk.drbd1584.out0 disk.drbd1552.out0 disk.drbd2078.in0 disk.drbd1931.in0 disk.drbd1931.out0 disk.drbd1938.in0 disk.drbd1510.out0 disk.drbd2111.in0 disk.sda1.in0 disk.drbd1550.out0 disk.sda.in0$ disk./.usageGB578.3444595336914 disk.drbd1584.in0 disk.drbd2111.out0 disk.drbd1828.in0 network.sent 7.695999e+06 disk.drbd1226.out0! proc.memory.rssMB 11260.40625* proc.memory.percent17.182016372680664 disk.drbd1356.out0 disk.drbd1285.out0 disk.nvme0n1.in 2.47265625 disk.drbd1356.in0 disk.drbd1673.out0 disk.drbd1285.in0) proc.memory.availableMB 38508.9765625 proc.cpu.threads29 network.recv492419 disk.dm-0.in 2.47265625 disk.drbd1828.out0 disk.sda1.out0 disk.dm-0.out170.2880859375 disk.drbd1817.in0 disk.rbd1.in0 cpu13.810875786060667) disk./.usagePercent32.34404978233396 disk.sda2.in0 disk.drbd1817.out0 disk.drbd1550.in0 disk.drbd1289.in0 disk.drbd1938.out0 disk.drbd2118.out0 disk.rbd0.in0 disk.drbd1510.in0% memory_percent41.239964962005615 disk.rbd0.out0 disk.drbd1965.in0 disk.drbd1965.out0 disk.drbd1226.in0# disk.nvme0n1.out170.2880859375 disk.drbd2078.out0 disk.drbd2118.in0 disk.rbd1.out14.9375 disk.drbd1673.in0 disk.drbd1552.in0 disk.sda2.out 102.95703125 disk.sda.out 102.95703125 disk.drbd2041.in0 disk.drbd2041.out0 disk.drbd1289.out02- _wandbruntime3752  _runtime37527 _step672 ' lr-AdamW/pg10.00021774920361435123 ' lr-AdamW/pg20.00021774920361435123 trainer/global_step8449 % _timestamp1.7857990395030649e+09 _runtime3752.326024695} % _timestamp1.7857990395030649e+09 _runtime3752.326024695 _step672 ' lr-AdamW/pg10.00021774920361435123 ' lr-AdamW/pg20.00021774920361435123 trainer/global_step8449]>I2- _wandbruntime3752  _runtime3752ǯ'O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 7636026887168 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated33270047 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed148875747990744 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed57474541 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated145405986198528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 7416833012396 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed145405961294848 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated89238643 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated57474801 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 7434943238656 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated145063690097664 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 % _timestamp1.7857990395039897e+09 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed145063673058304 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current258 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated141459017820832 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated152498720765952 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed153041988182016 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 7434943238656 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated56137929 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7636159658496 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed152498616296960 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated145063690097664 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 _step673 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 _runtime 3752.32724582 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed152498616296960 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed33270045 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current260 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 145366914 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed56137927 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed24204496 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7435030668288 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed89228987 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated152498720765952 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 145376572 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed89228987 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 7416747209272 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed57474541 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated145405986198528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 7416833012396 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed145405961294848 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated89238643 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated57474801 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 7434943238656 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated145063690097664 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 % _timestamp1.7857990395039897e+09 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed145063673058304 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current258 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated141459017820832 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated152498720765952 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed153041988182016 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 7434943238656 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated56137929 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7636159658496 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed152498616296960 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated145063690097664 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 _step673 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 _runtime 3752.32724582 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed152498616296960 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed33270045 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current260 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 145366914 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed56137927 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed24204496 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640)I}2- _wandbruntime3752  _runtime3752q M QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 124375552 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 % _timestamp1.7857990398637087e+09 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 7435910295040 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated56144572 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed145080839821312 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 trainer/global_step8449 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed24207305 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed145423168411648 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed89239510 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 145384080 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated33273984 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7417624682494 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _step674 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 145393752 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 145393752 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated89249180 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed152516662679552 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed148893365636286 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current229 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated141475757993152 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed141475740953792 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 7417710485730 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed56144570 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated152516767155712 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed57481287 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated24207532 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 149279232 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed153060103343616 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed33273982 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 7636934931968 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated148893468478882 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7637059307520 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed145080839821312 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed145080839821312 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed89239510 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 145384080 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated33273984 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7417624682494 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _step674 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 145393752 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 145393752 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated89249180 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed152516662679552 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed148893365636286 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current229 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated141475757993152 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed141475740953792 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 7417710485730 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed56144570 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated152516767155712 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed57481287 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated24207532 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 149279232 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed153060103343616 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed33273982 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 7636934931968 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated148893468478882 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7637059307520 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 FDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated145922028248064 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed24348393 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated56470079 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 7460721475904 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 _step677 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated56470079 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed153400950582784 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated149756747912736 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7681374105600 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 146227066 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed89756989 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed153400950582784 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated153401055051776 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 $ _timestamp1.785799061311855e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated89766645 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current239 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated89766645 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated24348632 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed149756645070252 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed146266317134848 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 7681239237120 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated145922028248064 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7479026803712 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed56470077 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current241 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed145922011208704 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 7479026803712 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed04*zO WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed149756645070252 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed146266317134848 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 7681239237120 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated145922028248064 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7479026803712 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed56470077 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current241 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed145922011208704 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 7479026803712 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 trainer/global_step8499 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 UGDӹ eviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed153947556371968 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed89756989 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated57815529 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated142296026436832 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated145922028248064 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed24348393 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated56470079 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 7460721475904 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 _step677 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated56470079 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed153400950582784 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated149756747912736 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7681374105600 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 146227066 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed89756989 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed153400950582784 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated153401055051776 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 $ _timestamp1.785799061311855e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated89766645 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current239 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated89766645 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated24348632 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888fF2- _wandbruntime3774  _runtime3774 h&M VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated33470834 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed146283524251648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 trainer/global_step8499 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 146244232 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated145939195011072 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 146253904 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated153965825031680 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed149774262715794 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed145939177971712 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed33470832 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated89777182 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed56476720 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 141152768 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed89767512 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 146253904 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated56476722 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 7479906430464 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 166056448 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 7461598949238 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 7682134723584 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7682275876352 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated145939195011072 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 146244232 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 _step678 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed153418996965376 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed89767512 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated149774365558390 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 7479818993664 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current263 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated89777182 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 _runtime3774.509431439 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated56476722 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated24351487 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 # _timestamp1.78579906168611e+09 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed57822056 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136BV&M C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed142312749569792 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7461513146002 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated153419101441536 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current265 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated146283549155328 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed56476720 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 _step678 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed153418996965376 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed89767512 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated149774365558390 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 7479818993664 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current263 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated89777182 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 _runtime3774.509431439 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated56476722 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated24351487 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 # _timestamp1.78579906168611e+09 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed57822056 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated33470834 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed146283524251648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 trainer/global_step8499 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 146244232 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated145939195011072 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 146253904 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated153965825031680 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed149774262715794 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed145939177971712 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed33470832 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated89777182 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed56476720 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 141152768 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed89767512 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 146253904 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated56476722 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 7479906430464 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 166056448 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 7461598949238 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 7682134723584 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7682275876352 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated145939195011072 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 146244232 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 7726400258560 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 _runtime3796.069803943 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7523022939136 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 7522935509504 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed154303284868608 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated90294647 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated58155707 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 147096876 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated90294647 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 7504524136288 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed146780349359104 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated147126697878528 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7726530932736 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed154303284868608 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated146780366398464 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated24491960 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 7523022939136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed154853073233408 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated56802229 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed58155469 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed24491724 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130OV'O O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated56802229 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 7726400258560 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 _runtime3796.069803943 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7523022939136 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 7522935509504 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed154303284868608 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated90294647 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated58155707 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 147096876 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated90294647 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 7504524136288 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed146780349359104 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated147126697878528 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7726530932736 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed154303284868608 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated146780366398464 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated24491960 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 7523022939136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed154853073233408 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated56802229 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed58155469 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed24491724 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024'22- _wandbruntime3796  _runtime3796khA&M E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 147104384 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated58162482 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 7727302609920 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed33667682 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed146797516122112 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed90295514 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 147114056 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated33667684 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated150655262637898 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated24494798 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated56808872 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed58162224 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7505401609510 _step682 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated146797533161472 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated147143904995328 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated154321435727360 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated143149775225152 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated90305184 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed56808870 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated90305184 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed154871182701568 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed147143880091648 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed150655159795302 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 147114056 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed146797516122112 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 147104384 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated58162482 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 7727302609920 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed33667682 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed146797516122112 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed90295514 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 147114056 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated33667684 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated150655262637898 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated24494798 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated56808872 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed58162224 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7505401609510 _step682 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated146797533161472 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated147143904995328 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated154321435727360 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated143149775225152 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated90305184 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed56808870 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated90305184 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed154871182701568 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed147143880091648 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed150655159795302 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 147114056 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed146797516122112 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 EDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed147987028814848 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated151518542071752 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 7548498402920 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current254 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated58495919 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated147987053718528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated143970043668832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated155205723623424 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed155205619154432 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed58495665 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated33860597 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated90822649 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656-b"O ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed155758597023232 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated147638704548864 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7567019074560 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7771696785408 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 $ _timestamp1.785799105518397e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed57134377 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed147987028814848 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated151518542071752 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 7548498402920 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current254 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated58495919 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated147987053718528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated143970043668832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated155205723623424 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed155205619154432 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed58495665 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated33860597 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated90822649 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 E?DeviceStatsMonitor.on_6train_batch_start/oversize_segments.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated58502719 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated148004260835328 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated90833186 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated90833186 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current259 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 7567898701312 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 147974208 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current261 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 _step686 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 147964536 trainer/global_step8599 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 7772467422208 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated147655871311872 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 147974208 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated33864534 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 7549375876254 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed90823516 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated147655871311872 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed33864532 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated155776865216000 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated151536159717406 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed143986766801792 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed155223665537024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated57141022 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed148004235931648 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7549290073018 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 7567811264512 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed58502458 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 % _timestamp1.7857991058909378e+09 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 147964536 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated155223770013184 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed90823516 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 _runtime3818.714284995 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated24638185 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed151536056874810 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7772604380672 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated147655871311872 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 147974208 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated33864534 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 7549375876254 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed90823516 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated147655871311872 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed33864532 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated155776865216000 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated151536159717406 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed143986766801792 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed155223665537024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated57141022 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed148004235931648 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7549290073018 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 7567811264512 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed58502458 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 % _timestamp1.7857991058909378e+09 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 147964536 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated155223770013184 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed90823516 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 _runtime3818.714284995 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated24638185 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed151536056874810 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7772604380672 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated58502719 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated148004260835328 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated90833186 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated90833186 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current259 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 7567898701312 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 147974208 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current261 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 _step686 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 147964536 trainer/global_step8599#K2- _wandbruntime3818  _runtime3818) trainer/global_step8599 _runtime3818.715452322 _step687 2loss/SRqqt1/assignment_loss0.8691956266252832 1loss/SRqqt2/assignment_loss1.042759326852283 /loss/SRqqt1/detection_loss1.89765044267882 &loss/total_loss19.867234746044232 0loss/FRt1/assignment_loss3.2434084692471465 0loss/FRt2/assignment_loss2.4643827567155334 4loss/FBt2/assignment_loss0.00011075021994007746 .loss/FRt2/detection_loss2.347816724030451 0loss/SRqqt2/detection_loss1.088457738518115 /loss/FBt1/detection_loss0.8403291766545876 .loss/FBt2/detection_loss3.278370509545008 % _timestamp1.7857991058925703e+09 4loss/FBt1/assignment_loss1.3101309027497406e-05 epoch0 .loss/FRt1/detection_loss2.7947401236480397&  .loss/FBt2/detection_loss3.278370509545008 % _timestamp1.7857991058925703e+09 4loss/FBt1/assignment_loss1.3101309027497406e-05 epoch0 .loss/FRt1/detection_loss2.794740123648039 trainer/global_step8599 _runtime3818.715452322 _step687 2loss/SRqqt1/assignment_loss0.8691956266252832 1loss/SRqqt2/assignment_loss1.042759326852283 /loss/SRqqt1/detection_loss1.89765044267882 &loss/total_loss19.867234746044232 0loss/FRt1/assignment_loss3.2434084692471465 0loss/FRt2/assignment_loss2.4643827567155334 4loss/FBt2/assignment_loss0.00011075021994007746 .loss/FRt2/detection_loss2.347816724030451 0loss/SRqqt2/detection_loss1.088457738518115 /loss/FBt1/detection_loss0.8403291766545876)Z(:  U disk.sda2.in0 disk.drbd1550.in0 disk.drbd1817.in0 disk.drbd1817.out0 disk.drbd1931.out0 disk.drbd1285.out0! disk.dm-0.out171.37255859375 proc.cpu.threads29 disk.drbd1356.out0 disk.drbd2111.out0 disk.drbd1552.out0 disk.sda1.in0 disk.drbd1584.in0! proc.memory.rssMB 11260.40625 disk.drbd2118.out0 disk.drbd2041.in0 disk.sda.out 105.11328125 disk.drbd1226.out0 disk.drbd2041.out0 disk.drbd1285.in0 disk.drbd1510.out0 disk.drbd2078.in0 disk.drbd2078.out0% memory_percent41.239988803863525 disk.drbd1673.in0 disk.drbd1673.out0 disk.drbd1828.in0 disk.sda1.out0$ disk.nvme0n1.out171.37255859375 disk.drbd1356.in0 disk.drbd1289.in0 disk.sda.in0 disk.nvme0n1.in 2.47265625 disk.rbd1.in0 disk.drbd1938.in0 network.recv499077* disk./.usagePercent32.344051062361004 disk.drbd2111.in0 disk.drbd1828.out0 disk.rbd0.in0) proc.memory.availableMB 38508.9609375 disk.sda2.out 105.11328125 disk.drbd1965.in0 disk.drbd1931.in0 disk.drbd1552.in0 disk.rbd1.out 15.1259765625 disk.drbd1965.out0 network.sent 7.837247e+06 disk.drbd1938.out0 disk.drbd1226.in0* proc.memory.percent17.182016372680664 disk.drbd1550.out0 disk.drbd1289.out0 disk.dm-0.in 2.47265625 disk.rbd0.out0 cpu13.8061681518605# disk./.usageGB578.344482421875 disk.drbd2118.in0 disk.drbd1510.in0 disk.drbd1584.out0ED(: ɧU gpu.0.gpu37.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts126.238' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.07933333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0+@(: ׺U gpu.0.gpu32.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts95.749' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent31.91633333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0}U"[(:  U disk.drbd1817.in0 disk.drbd1550.out0 disk.drbd1552.out0! proc.memory.rssMB 11260.40625 disk.drbd1285.out0 disk.sda1.in0 disk.drbd1584.out0) disk./.usagePercent32.34402780853635 disk.drbd2118.in0 disk.nvme0n1.in 2.47265625 disk.drbd1673.in0 disk.drbd1289.out0 disk.rbd0.out0! disk.dm-0.out171.69970703125 disk.drbd1226.out0 disk.rbd1.out 15.251953125 disk.drbd2041.in0 disk.sda.in0 disk.sda2.out 105.59765625 disk.drbd1938.in0* proc.memory.percent17.182016372680664 network.sent 7.882264e+06 disk.drbd1285.in0 disk.drbd2041.out0 disk.drbd1552.in0 network.recv500716 disk.dm-0.in 2.47265625 disk.drbd1226.in0 disk.drbd1965.in0 disk.drbd2078.out0 disk.drbd1584.in0$ disk.nvme0n1.out171.69970703125 disk.drbd1356.in0 disk.drbd2111.out0 disk.drbd1828.out0 cpu13.805216497436477# disk./.usageGB578.344066619873 disk.drbd2078.in0 disk.sda2.in0 disk.drbd2111.in0 disk.drbd1817.out0* proc.memory.availableMB38508.13671875 proc.cpu.threads29 disk.drbd1510.out0 disk.drbd2118.out0 disk.rbd1.in0 disk.drbd1550.in0 disk.drbd1965.out0 disk.drbd1931.in0 disk.drbd1931.out0 disk.drbd1828.in0 disk.drbd1289.in0 disk.drbd1356.out0 disk.drbd1673.out0 disk.drbd1938.out0% memory_percent41.241246461868286 disk.drbd1510.in0 disk.rbd0.in0 disk.sda.out 105.59765625 disk.sda1.out0|2- _wandbruntime3840  _runtime3840Zg _step688 ' lr-AdamW/pg10.00022290364091141242 ' lr-AdamW/pg20.00022290364091141242 trainer/global_step8649 % _timestamp1.7857991273097012e+09 _runtime3840.132673473LR _runtime3840.132673473 _step688 ' lr-AdamW/pg10.00022290364091141242 ' lr-AdamW/pg20.00022290364091141242 trainer/global_step8649 % _timestamp1.7857991273097012e+0962- _wandbruntime3840  _runtime3840 AO JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current257 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed57466527 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated156664272447488 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed34057445 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated148497042699264 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed148497025659904 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed91340995 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 7611015209984 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed156107953440256 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed156664108481024 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7611015209984 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated57466529 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_s:DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current255 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7816862888960 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 148817180 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 7592386866428 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated148497042699264 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated58836269 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed91340995 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 7816723826176 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _step689 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 7610927780352 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed58836012 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1E'O WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed144807035245472 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated156108057909248 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed24778567 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated34057447 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 148807522 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed156107953440256 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated148847409558528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 trainer/global_step8649 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated57466529 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated91350651 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 % _timestamp1.7857991273106282e+09 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed148497025659904 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed57466527 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed152399336308776 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated91350651 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed148847384654848 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated152399439151260 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 _runtime3840.135498821 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 7610927780352 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current255 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7816862888960 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 148817180 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 7592386866428 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated148497042699264 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated58836269 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed91340995 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 7816723826176 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _step689 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 7610927780352 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed58836012 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current257 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed57466527 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated156664272447488 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed34057445 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated148497042699264 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed148497025659904 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed91340995 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 7611015209984 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed156107953440256 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed156664108481024 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7611015209984 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated57466529 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 148834360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 7593264339762 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 148824688 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current254 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7593178536526 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated156126104299008 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed148864591771648 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed34061382 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated156682377927168 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated152417056796914 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated34061384 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated148864616675328 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed156125999822848 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated144823792457152 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 7611807399936 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed152416953954318 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed148514192422912 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 7611894836736 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed148514192422912 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 _step690 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated91361188 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated58842942 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 148824688 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 7817628487680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 7611807399936 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 $ _timestamp1.785799127681144e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed91351518 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _runtime3840.504471711 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated148514209462272 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed91351518 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed24781306 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed156682220259328 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed144823775417792 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed58842688 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 trainer/global_step8649 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated148514209462272 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 7611894836736 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated24781558 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640ܜVhM XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed144823775417792 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed58842688 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 trainer/global_step8649 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated148514209462272 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 7611894836736 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated24781558 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current252 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated156126104299008 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 148834360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 7593264339762 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 148824688 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current254 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7593178536526 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated156126104299008 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed148864591771648 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed34061382 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated156682377927168 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated152417056796914 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated34061384 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated148864616675328 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed156125999822848 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated144823792457152 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 7611807399936 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed152416953954318 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed148514192422912 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 7611894836736 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed148514192422912 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 A9DeviceStatsMonitor.on_1Ix train_batch_end/segment.all.current251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 _step690 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated91361188 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated58842942 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 148824688 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 7817628487680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 7611807399936 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 $ _timestamp1.785799127681144e+09 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed91351518 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _runtime3840.504471711 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated148514209462272 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed91351518 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed24781306 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed156682220259328k\2- _wandbruntime3840  _runtime3840  1loss/SRqqt2/detection_loss1.0471196257449844 trainer/global_step8649 % _timestamp1.7857991276827679e+09 0loss/FRt1/assignment_loss3.0438359158762385 2loss/SRqqt1/assignment_loss0.9917564812675118 2loss/SRqqt2/assignment_loss1.0449550933475402 4loss/FBt2/assignment_loss2.9577639816125155e-05 .loss/FBt2/detection_loss1.572721260599792 %loss/total_loss18.36574501767609 0loss/FRt2/assignment_loss2.4021516517854073 4loss/FBt1/assignment_loss0.00035208606806361664 .loss/FRt1/detection_loss2.608321041873332 .loss/FRt2/detection_loss2.680342640465414 1loss/SRqqt1/detection_loss1.9034865428507328 _step691 /loss/FBt1/detection_loss1.0706731001572574 epoch0 _runtime3840.505603011 1loss/SRqqt2/detection_loss1.0471196257449844 trainer/global_step8649 % _timestamp1.7857991276827679e+09 0loss/FRt1/assignment_loss3.0438359158762385 2loss/SRqqt1/assignment_loss0.9917564812675118 2loss/SRqqt2/assignment_loss1.0449550933475402 4loss/FBt2/assignment_loss2.9577639816125155e-05 .loss/FBt2/detection_loss1.572721260599792 %loss/total_loss18.36574501767609 0loss/FRt2/assignment_loss2.4021516517854073 4loss/FBt1/assignment_loss0.00035208606806361664 .loss/FRt1/detection_loss2.608321041873332 .loss/FRt2/detection_loss2.680342640465414 1loss/SRqqt1/detection_loss1.9034865428507328 _step691 /loss/FBt1/detection_loss1.0706731001572574 epoch0 _runtime3840.505603011r(: ȡU gpu.0.gpu34.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts96.246' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent32.082 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors05Y(:  ӜU disk.dm-0.in 2.47265625 disk.drbd2041.out0 disk.drbd1550.in0 disk.drbd1938.in0! disk.dm-0.out171.89892578125 disk.drbd2118.in0$ memory_percent41.24134182929993 network.sent 7.885288e+06 disk.rbd0.in0 cpu13.804239690553569) disk./.usagePercent32.34402972857692 disk.drbd1510.out0$ disk.nvme0n1.out171.89892578125 disk.rbd1.out 15.251953125 disk.drbd1965.out0 disk.drbd1550.out0 disk.drbd1938.out0 disk.drbd1285.out0 disk.drbd1226.out0 disk.drbd1584.in0 disk.drbd1965.in0 disk.drbd1552.in0 disk.drbd1828.out0* proc.memory.availableMB38508.07421875 disk.sda.in0 disk.nvme0n1.in 2.47265625 disk.drbd1817.in0 disk.sda1.out0 disk.drbd2118.out0 disk.sda2.in0* proc.memory.percent17.182016372680664$ disk./.usageGB578.3441009521484 disk.drbd1226.in0 disk.drbd1356.out0 disk.drbd2111.in0! proc.memory.rssMB 11260.40625 disk.rbd0.out0 disk.drbd1285.in0 disk.drbd1673.out0 disk.drbd1931.out0 network.recv501641 disk.sda1.in0 disk.rbd1.in0 proc.cpu.threads29 disk.drbd2111.out0 disk.drbd1552.out0 disk.drbd1931.in0 disk.drbd1510.in0 disk.drbd1356.in0 disk.drbd1673.in0 disk.drbd1289.in0 disk.drbd1289.out0 disk.drbd2078.out0 disk.sda2.out 105.8046875 disk.drbd1817.out0 disk.drbd2041.in0 disk.drbd1828.in0 disk.sda.out 105.8046875 disk.drbd2078.in0 disk.drbd1584.out0?ŝ2- _wandbruntime3862  _runtime3862y2 trainer/global_step8699 % _timestamp1.7857991492521517e+09 _runtime 3862.07512681 _step692 ' lr-AdamW/pg10.00022419225023567776 ' lr-AdamW/pg20.000224192250235677769l % _timestamp1.7857991492521517e+09 _runtime 3862.07512681 _step692 ' lr-AdamW/pg10.00022419225023567776 ' lr-AdamW/pg20.00022419225023567776 trainer/global_step8699*ˉ2- _wandbruntime3862  _runtime3862JRY'O QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated59176859 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed149707740494848 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 7636275329936 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated24922562 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated157010392195072 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated57798679 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated149355380849664 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated57798679 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed157010287726080 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 7655011345408 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated157010392195072 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7861993081344 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated149355380849664 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed57798677 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 _step693 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed145644043861472 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed91868997 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 7654923915776 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 7636189526812 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed149355363810304 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed157569604999168 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 7654923915776 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 trainer/global_step8699 % _timestamp1.7857991492531393e+09 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed24922325 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032Z'O YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated157010392195072 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7861993081344 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated149355380849664 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed57798677 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 _step693 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed145644043861472 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed91868997 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 7654923915776 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 7636189526812 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed149355363810304 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed157569604999168 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 7654923915776 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 trainer/global_step8699 % _timestamp1.7857991492531393e+09 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed24922325 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated59176859 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed149707740494848 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 7636275329936 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated24922562 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated157010392195072 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated57798679 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated149355380849664 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated57798679 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed157010287726080 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 7655011345408 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 2- _wandbruntime3862  _runtime3862,!M L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 149694512 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed91879520 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 7862771254784 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed149372530573312 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current249 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 7655803535360 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed91879520 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated149372547612672 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 149684840 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed57805320 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated34258234 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated157028438584832 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated149372547612672 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 149694512 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7862906116096 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 trainer/global_step8699 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed59183431 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated57805322 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed149372530573312 REDeviceStatsMonitor.on_Kbtrain_batch_end/allocation.small_pool.allocated91889190 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 149684840 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated153297953876422 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 _runtime3862.447763953 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 7655890972160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated157028438584832 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed34258232 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed57805320 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated59183682 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 % _timestamp1.7857991496244345e+09 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated157587878631424 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed149724947611648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed157587718866432 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated145660801073152 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 _step694 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed153297851033826 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated91889190 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392d~&M ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed145660784033792 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed57805320 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated59183682 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 % _timestamp1.7857991496244345e+09 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated157587878631424 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed149724947611648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed157587718866432 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated145660801073152 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 _step694 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed153297851033826 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated91889190 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 149694512 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed91879520 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 7862771254784 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed149372530573312 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current249 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 7655803535360 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed91879520 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated149372547612672 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 149684840 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed57805320 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated34258234 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated157028438584832 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated149372547612672 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 149694512 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7862906116096 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 trainer/global_step8699 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed59183431 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated57805322 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed149372530573312 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated91889190 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 149684840 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated153297953876422 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 _runtime3862.447763953 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 7655890972160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated157028438584832 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed34258232 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 EDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated157912726480896 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated150213719000064 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7699007480832 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 _runtime3884.099845481 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current248 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 7698920051200 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed59517171 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed150213701960704 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 % _timestamp1.7857991712765772e+09 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed157912622011904 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated157912726480896 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated146481069516832 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360v?O WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated58130829 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated92406655 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed158475124761600 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated150568121238528 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7907163295232 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated158475284533760 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed92396999 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated25066272 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current246 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed59517171 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed150213701960704 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 % _timestamp1.7857991712765772e+09 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed157912622011904 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated157912726480896 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated146481069516832 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 150537484 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 7680077990320 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated157912726480896 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated150213719000064 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7699007480832 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 _runtime3884.099845481 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current248 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 7698920051200 J6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 _step698 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed150230868723712 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated157930772870656 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 150554664 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 150544992 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated158493393585152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7680955463542 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed34455082 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated154178850955930 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed154178748113334 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 7907932465664 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated92417192 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated34455084 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed58137470 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 7699887107584 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed59523992 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated157930772870656 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated150230885763072 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated58137472 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated150585328355328 $ _timestamp1.785799171645871e+09 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current260 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated25069168 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 7699799670784 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 150554664 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7908065229824 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 7681041266778 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated146497809689152 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated59524252 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed150585303451648 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated92417192 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 7699799670784 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed146497792649792 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated150230885763072 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed158493235917312 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 trainer/global_step8749 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 _step698 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed150230868723712 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated157930772870656 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 150554664 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 150544992 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated158493393585152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7680955463542 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed34455082 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated154178850955930 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed154178748113334 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 7907932465664 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated92417192 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated34455084 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed58137470 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 7699887107584 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed59523992 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated157930772870656 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated150230885763072 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated58137472 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated150585328355328 $ _timestamp1.785799171645871e+09 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current260 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated25069168 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 7699799670784 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 150554664 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7908065229824 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 7681041266778 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated146497809689152 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated59524252 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed150585303451648 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated92417192 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 7699799670784 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed146497792649792 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated150230885763072 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed158493235917312 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 trainer/global_step8749 FDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed59857129 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated25209392 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed158814956297728 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 7724052256952 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 7952227747328 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed151072040111104 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed155042027547300 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 trainer/global_step8799 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed34647995 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 151397636 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated159380841791488 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed159380679922176 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 7743003616256 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated147318078132832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated59857389 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated158815060766720 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated158815060766720 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 7742916186624 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 151387978 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed158814956297728 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed58462977 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 _runtime3906.317175001 _step701 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed151428452174848 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712ᾋ'O L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed92925001 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated92934657 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current260 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7952364712960 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7743003616256 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated151072057150464 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 7742916186624 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed59857129 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated25209392 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed158814956297728 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 7724052256952 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 7952227747328 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed151072040111104 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed155042027547300 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 trainer/global_step8799 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed34647995 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 151397636 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated159380841791488 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed159380679922176 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 7743003616256 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated147318078132832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated59857389 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated158815060766720 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated158815060766720 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 7742916186624 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 151387978 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed158814956297728 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed58462977 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 _runtime3906.317175001 _step701 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed151428452174848 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated151428477078528Bv2- _wandbruntime3906  _runtime39063PX&M E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed92935524 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed155059645192842 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current254 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated147334818305152 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 trainer/global_step8799 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 151405144 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed34651932 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated151445684195328 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 7743795806208 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed158833002680320 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 151414816 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed59863907 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed92935524 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 7743883243008 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed159398787907584 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7953263477248 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 151414816 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 7743795806208 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed147334801265792 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated92945194 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated151089223913472 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed151089206874112 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed151445659291648 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated25212227 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 _runtime3906.681183735 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed58469620 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated158833107156480 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated158833107156480 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed25211975 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 $ _timestamp1.785799193857896e+09 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 151405144 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887#; M J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed92935524 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 7743883243008 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed159398787907584 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7953263477248 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 151414816 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 7743795806208 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed147334801265792 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated92945194 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated151089223913472 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed151089206874112 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed151445659291648 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated25212227 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 _runtime3906.681183735 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed58469620 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated158833107156480 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytesqh>.large_pool.peak86278144 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated158833107156480 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed25211975 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 $ _timestamp1.785799193857896e+09 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 151405144 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed92935524 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed155059645192842 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current254 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated147334818305152 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 trainer/global_step8799 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 151405144 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed34651932 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated151445684195328 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 7743795806208 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed158833002680320 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 151414816 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed59863907 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392;2- _wandbruntime3906  _runtime3906 1loss/SRqqt1/detection_loss1.7564104582130347 epoch0 trainer/global_step8799 _step703 1loss/FBt2/assignment_loss0.05462608932429021 /loss/FRt1/detection_loss2.4695879132265137 /loss/FRt1/assignment_loss2.645135676576978 2loss/SRqqt1/assignment_loss0.8312773972152258 2loss/SRqqt2/assignment_loss0.8784309260857602 1loss/SRqqt2/detection_loss0.9103273041972441 /loss/FBt2/detection_loss2.8328308880329134 &loss/total_loss17.681075608889692 % _timestamp1.7857991938590677e+09 0loss/FRt2/assignment_loss2.3765370352293855 3loss/FBt1/assignment_loss0.0025846352296228225 /loss/FRt2/detection_loss2.2945223251978555 .loss/FBt1/detection_loss0.628804960360867 _runtime3906.681973329W  epoch0 trainer/global_step8799 _step703 1loss/FBt2/assignment_loss0.05462608932429021 /loss/FRt1/detection_loss2.4695879132265137 /loss/FRt1/assignment_loss2.645135676576978 2loss/SRqqt1/assignment_loss0.8312773972152258 2loss/SRqqt2/assignment_loss0.8784309260857602 1loss/SRqqt2/detection_loss0.9103273041972441 /loss/FBt2/detection_loss2.8328308880329134 &loss/total_loss17.681075608889692 % _timestamp1.7857991938590677e+09 0loss/FRt2/assignment_loss2.3765370352293855 3loss/FBt1/assignment_loss0.0025846352296228225 /loss/FRt2/detection_loss2.2945223251978555 .loss/FBt1/detection_loss0.628804960360867 _runtime3906.681973329 1loss/SRqqt1/detection_loss1.7564104582130347OY(: ؚU gpu.0.gpu28.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts128.162' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent42.720666666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0 D(:  ͂V disk.drbd1289.out0 disk.drbd2078.out0 disk.nvme0n1.in 2.47265625 disk.sda2.in0 disk.drbd2111.out0 network.sent 8.023159e+06 disk.drbd1226.in0 disk.drbd1965.in0 disk.drbd1817.in0 disk.drbd1226.out0 disk.drbd2118.in0 disk.drbd2118.out0 network.recv507101) disk./.usagePercent32.34404316886089 disk.sda.out 107.47265625 disk.drbd1285.in0 disk.dm-0.out174.25 disk.nvme0n1.out174.25 disk.rbd1.in0 disk.drbd1673.out0 disk.sda2.out 107.47265625 disk.drbd1931.out0% proc.memory.availableMB 38508.125 disk.drbd1550.in0 disk.sda1.in0 disk.dm-0.in 2.47265625 disk.drbd1673.in0 disk.drbd1552.out0$ memory_percent41.24126434326172 disk.drbd1510.in0 disk.drbd1356.in0 disk.drbd1356.out0 proc.cpu.threads29 disk.drbd1550.out0 disk.drbd1931.in0 disk.rbd0.out0 disk.drbd1584.out0 disk.drbd2041.out0 cpu13.800721675069356 disk.sda1.out0 disk.drbd1510.out0 disk.drbd1552.in0 disk.drbd1938.in0 disk.sda.in0 disk.drbd1584.in0 disk.drbd1828.in0* proc.memory.percent17.182016372680664 disk.drbd1938.out0 disk.drbd2111.in0 disk.drbd1817.out0 disk.drbd2041.in0 disk.drbd1828.out0 disk.rbd0.in0 disk.drbd2078.in0 disk.rbd1.out 15.556640625$ disk./.usageGB578.3443412780762 disk.drbd1289.in0 disk.drbd1285.out0 disk.drbd1965.out0! proc.memory.rssMB 11260.40625Z2- _wandbruntime3928  _runtime3928#9yX ' lr-AdamW/pg10.00022805807820847366 ' lr-AdamW/pg20.00022805807820847366 trainer/global_step8849 $ _timestamp1.785799215206332e+09 _runtime3928.029306542 _step704 /g ' lr-AdamW/pg10.00022805807820847366 ' lr-AdamW/pg20.00022805807820847366 trainer/global_step8849 $ _timestamp1.785799215206332e+09 _runtime3928.029306542 _step7041R2- _wandbruntime3928  _runtime3928. 'O C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 7767940720460 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current260 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7786999751680 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed159717290583552 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed155922924626808 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated159717395052544 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 _runtime3928.030614339 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 _step705 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7997517444096 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated93462659 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated148155086748832 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed151930378261504 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 7786999751680 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated160286350362624 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated152288832918528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 7767854917336 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed93453003 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed160286184299008 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 7786912322048 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed159717290583552 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 7997376284160 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated155923027469292 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed151930378261504 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated34844847 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated151930395300864 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current258 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated25353298 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 % _timestamp1.7857992152072947e+09 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current260 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7786999751680 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed159717290583552 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed155922924626808 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated159717395052544 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 _runtime3928.030614339 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 _step705 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 7997517444096 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated93462659 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated148155086748832 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed151930378261504 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 7786999751680 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated160286350362624 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated152288832918528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 7767854917336 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed93453003 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed160286184299008||2- _wandbruntime3928  _runtime3928qM XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated58801772 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated155940645114946 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed148171809881792 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current246 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed155940542272350 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 JBDevic 3p&#eStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 152274968 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated159735441442304 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed159735336966144 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed93463526 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed58801770 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 7787879378432 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated151947562063872 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed93463526 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 152265296 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7768732390558 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed152306015131648 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated93473196 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed60204644 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated160304451492352 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated159735441442304 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated151947562063872 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7998411457024 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated148171826921152 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 trainer/global_step8849 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime3928.414935797 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 152265296 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 7787879378432 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed34848782 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated25356108 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed25355862 _step706 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated58801772 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed151947545024512 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 % _timestamp1.7857992155914943e+09 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 7787791941632 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 7787791941632 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated152306040035328 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 7768818193794 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 152274968 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated93473196 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current248PS%&M H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 152274968 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated159735441442304 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed159735336966144 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed93463526 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed58801770 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 7787879378432 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated151947562063872 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed93463526 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 152265296 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7768732390558 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed152306015131648 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated93473196 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed60204644 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated160304451492352 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated159735441442304 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated151947562063872 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 7998411457024 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated148171826921152 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 trainer/global_step8849 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime3928.414935797 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 152265296 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 7787879378432 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed34848782 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated25356108 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed25355862 _step706 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated58801772 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed151947545024512 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 % _timestamp1.7857992155914943e+09 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 7787791941632 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 7787791941632 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated152306040035328 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 7768818193794 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 152274968 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated93473196 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current248 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated58801772 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated155940645114946 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed148171809881792 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current246 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed155940542272350 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated13062- _wandbruntime3928  _runtime3928SB _step707 3loss/FBt2/assignment_loss0.0003867628684846832 /loss/FRt1/detection_loss2.4192368494139775 0loss/SRqqt1/detection_loss2.024387498230984 /loss/FBt2/detection_loss2.0851219675757666 /loss/FRt1/assignment_loss2.972850471072727 0loss/FRt2/assignment_loss2.3468958474695683 2loss/SRqqt1/assignment_loss1.3575328596246738 2loss/SRqqt2/assignment_loss0.8663555245696833 % _timestamp1.7857992155937595e+09 epoch0 %loss/total_loss18.52480441573245 3loss/FBt1/assignment_loss0.0016274941071172419 /loss/FRt2/detection_loss2.5052270074374974 0loss/SRqqt2/detection_loss0.936987501781966 /loss/FBt1/detection_loss1.0081946315800048 trainer/global_step8849 _runtime 3928.41664191쑮:  % _timestamp1.7857992155937595e+09 epoch0 %loss/total_loss18.52480441573245 3loss/FBt1/assignment_loss0.0016274941071172419 /loss/FRt2/detection_loss2.5052270074374974 0loss/SRqqt2/detection_loss0.936987501781966 /loss/FBt1/detection_loss1.0081946315800048 trainer/global_step8849 _runtime 3928.41664191 _step707 3loss/FBt2/assignment_loss0.0003867628684846832 /loss/FRt1/detection_loss2.4192368494139775 0loss/SRqqt1/detection_loss2.024387498230984 /loss/FBt2/detection_loss2.0851219675757666 /loss/FRt1/assignment_loss2.972850471072727 0loss/FRt2/assignment_loss2.3468958474695683 2loss/SRqqt1/assignment_loss1.3575328596246738 2loss/SRqqt2/assignment_loss0.8663555245696833(: U gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts120.171' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent40.057 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0T(:  ՕV disk.rbd1.out 15.712890625 disk.drbd1584.out0 disk.sda2.out 107.83984375 disk.sda1.in0 disk.nvme0n1.out 174.8671875 disk.drbd1817.out0 disk.drbd1550.out0 disk.drbd2118.in0 disk.drbd1938.out0 disk.rbd0.in0 disk.drbd1584.in0 disk.drbd2111.in0 cpu13.799717302766972 proc.cpu.threads29) disk./.usagePercent32.34426248016129 disk.drbd1289.in0 disk.drbd1938.in0 disk.drbd1226.in0 disk.drbd2041.in0 disk.drbd1817.in0% memory_percent41.241687536239624 disk.drbd1356.out0 disk.sda2.in0! proc.memory.rssMB 11260.40625 network.recv508855 disk.drbd1550.in0 disk.drbd1226.out0 disk.drbd1965.in0 disk.drbd1931.in0 disk.drbd1285.in0 disk.drbd2041.out0 disk.sda.in0 disk.drbd1510.out0 disk.drbd2078.in0* proc.memory.percent17.182016372680664 disk.drbd1673.in0 disk.drbd1931.out0 disk.sda.out 107.83984375 disk.sda1.out0 disk.dm-0.out 174.8671875 disk.drbd1356.in0 disk.drbd2111.out0 disk.drbd1828.out0 disk.rbd0.out0 disk.rbd1.in0 disk.dm-0.in 2.47265625 disk.drbd2078.out0 network.sent 8.068156e+06$ disk./.usageGB578.3482627868652 disk.drbd1552.out0 disk.drbd1289.out0 disk.drbd2118.out0 disk.drbd1965.out0 disk.drbd1828.in0 disk.drbd1510.in0 disk.nvme0n1.in 2.47265625* proc.memory.availableMB38507.84765625 disk.drbd1673.out0 disk.drbd1552.in0 disk.drbd1285.out0S>(: ډU gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts127.169' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.38966666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0Q(:  ϋU disk.drbd2111.in0 disk.drbd1673.out0 network.recv510638 disk.drbd1289.out0 disk.drbd1226.out0 disk.drbd2078.out0 disk.drbd1510.out0 disk.dm-0.in 2.47265625 disk.drbd1226.in0 disk.drbd1817.out0 cpu13.79887837073811$ disk./.usageGB578.3482818603516 disk.drbd1931.in0 disk.drbd1931.out0 disk.drbd1552.out0 disk.drbd1289.in0 disk.drbd2078.in0 disk.drbd2118.in0 disk.drbd2118.out0 disk.drbd2041.out0 disk.drbd1938.out0 disk.rbd0.in0 disk.drbd1285.out0* proc.memory.percent17.182016372680664 disk.drbd1550.out0' proc.memory.availableMB 38508.09375 disk.sda1.in0 disk.drbd1356.out0 disk.sda2.in0 disk.sda2.out 108.23828125 proc.cpu.threads29 disk.drbd1584.out0 disk.drbd1817.in0 disk.drbd1965.in0 disk.drbd2041.in0 disk.drbd1552.in0 disk.rbd0.out0 disk.sda.out 108.23828125 disk.rbd1.out 15.712890625 disk.dm-0.out 175.13671875 disk.drbd1828.in0 disk.drbd1550.in0 disk.sda1.out0 disk.drbd1510.in0 disk.drbd1356.in0 disk.rbd1.in0$ memory_percent41.24131202697754 network.sent 8.113311e+06) disk./.usagePercent32.34426354685049 disk.drbd1938.in0 disk.nvme0n1.in 2.47265625 disk.drbd1673.in0 disk.drbd1828.out0 disk.drbd1285.in0 disk.drbd1584.in0 disk.drbd2111.out0 disk.drbd1965.out0! disk.nvme0n1.out 175.13671875! proc.memory.rssMB 11260.40625 disk.sda.in0g:C2- _wandbruntime3950  _runtime3950 ' lr-AdamW/pg10.00022934668753273895 ' lr-AdamW/pg20.00022934668753273895 trainer/global_step8899 $ _timestamp1.785799237755091e+09 _runtime3950.579680084 _step7086 _runtime3950.579680084 _step708 ' lr-AdamW/pg10.00022934668753273895 ' lr-AdamW/pg20.00022934668753273895 trainer/global_step8899 $ _timestamp1.785799237755091e+092- _wandbruntime3950  _runtime3950- O VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated160619729338368 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated35041697 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated25495567 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed152788716411904 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated93990661 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed35041695 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 % _timestamp1.7857992377557793e+09 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 7811743380844 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed160619624869376 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated152788733451264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed161191697741312 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated60537264 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed153149163854848 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated93990661 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed148992078325472 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed160619624869376 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 153108282 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated59127279 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed59127277 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 7830908457472 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed59127277 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed156803821706316 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed93981005 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated59127279 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated152788733451264 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current266 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed25495303 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated152788733451264 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current266 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed25495303 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated35041697 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated25495567 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed152788716411904 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated93990661 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed35041695 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 % _timestamp1.7857992377557793e+09 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 7811743380844 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed160619624869376 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated152788733451264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed161191697741312 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated60537264 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed153149163854848 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated93990661 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed148992078325472 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed160619624869376 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 153108282 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated59127279 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed59127277 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 7830908457472 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed59127277 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed156803821706316 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed93981005 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated59127279 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640e2- _wandbruntime3950  _runtime3950F#&M O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed160637671251968 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated94001198 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed152805883174912 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 _runtime3950.947702582 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated152805900214272 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated160637775728128 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 8043569206784 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 153125448 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed59133920 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed93991528 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed153166370971648 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated59133922 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated161209965082112 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 153135120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated94001198 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated59133922 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 7831875513856 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 7831875513856 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 7831788077056 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed156821439351858 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated149008835537152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 trainer/global_step8899 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated160637775728128 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed60543829 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 7812706657302 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _step710 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 153135120 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 8043440636928 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated156821542194454 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed59133920 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current261ch&M UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated161209965082112 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 153135120 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated94001198 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated59133922 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 7831875513856 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 7831875513856 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 7831788077056 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed156821439351858 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated149008835537152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 trainer/global_step8899 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated160637775728128 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed60543829 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 7812706657302 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _step710 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 153135120 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 8043440636928 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated156821542194454 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed59133920 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current261 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed160637671251968 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated94001198 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed152805883174912 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 _runtime3950.947702582 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated152805900214272 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated160637775728128 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 8043569206784 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 153125448 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed59133920 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed93991528 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed153166370971648 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated59133922 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E?DeviceStatsMonitor.on_r.E2train_batch_end/active.large_pool.current22- _wandbruntime3950  _runtime3950^  %loss/total_loss19.23587786824269 trainer/global_step8899 _runtime3950.948514248 1loss/SRqqt1/detection_loss1.8951816969626658 /loss/FBt1/detection_loss0.7600136794822497 $ _timestamp1.785799238125599e+09 _step711 /loss/FRt1/assignment_loss2.854402194023132 /loss/FRt1/detection_loss2.4250097696483137 0loss/SRqqt2/detection_loss1.232172679691392 0loss/FRt2/assignment_loss2.5831809710053837 /loss/FRt2/detection_loss2.5799487930010345 /loss/FBt2/detection_loss3.0115016493946314 epoch0 2loss/SRqqt1/assignment_loss0.9731162222656043 2loss/SRqqt2/assignment_loss0.9213335437605736 4loss/FBt1/assignment_loss1.9868212951400388e-08 4loss/FBt2/assignment_loss1.6649139496394127e-05s T 0loss/FRt2/assignment_loss2.5831809710053837 /loss/FRt2/detection_loss2.5799487930010345 /loss/FBt2/detection_loss3.0115016493946314 epoch0 2loss/SRqqt1/assignment_loss0.9731162222656043 2loss/SRqqt2/assignment_loss0.9213335437605736 4loss/FBt1/assignment_loss1.9868212951400388e-08 4loss/FBt2/assignment_loss1.6649139496394127e-05 %loss/total_loss19.23587786824269 trainer/global_step8899 _runtime3950.948514248 1loss/SRqqt1/detection_loss1.8951816969626658 /loss/FBt1/detection_loss0.7600136794822497 $ _timestamp1.785799238125599e+09 _step711 /loss/FRt1/assignment_loss2.854402194023132 /loss/FRt1/detection_loss2.4250097696483137 0loss/SRqqt2/detection_loss1.232172679691392J"Ϝ(: ցU gpu.0.gpu29.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts119.545' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent39.848333333333336 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0Wܩ]X(:  U disk.drbd1965.out0* proc.memory.percent17.182016372680664 disk.drbd1550.in0 disk.drbd1817.in0 disk.drbd1965.in0 network.sent 8.116427e+06 disk.drbd2041.in0 disk.drbd1552.in0 disk.sda1.out0 disk.drbd1285.in0 disk.drbd1584.in0 disk.drbd2118.out0 disk.drbd1285.out0 disk.sda2.out 108.58984375 disk.drbd2111.out0 disk.drbd1817.out0 disk.rbd1.in0 disk.drbd1931.in0 disk.sda1.in0 proc.cpu.threads29 network.recv511712 disk.drbd1828.in0 disk.drbd1510.in0 disk.drbd1510.out0 disk.rbd1.out 15.869140625! disk.dm-0.out175.54052734375 disk.drbd1673.in0 disk.drbd2041.out0 disk.drbd1552.out0 disk.rbd0.out0 disk.drbd1226.in0 disk.drbd2118.in0 cpu13.797982687566012 disk.drbd1828.out0 disk.sda.out 108.58984375 disk.drbd2078.in0 disk.nvme0n1.in 2.47265625 disk.drbd1356.out0) disk./.usagePercent32.34427421374254 disk.drbd1356.in0 disk.drbd1938.in0 disk.drbd1938.out0 disk.sda.in0! proc.memory.rssMB 11260.40625 disk.drbd1226.out0 disk.drbd2111.in0$ disk./.usageGB578.3484725952148 disk.drbd1289.out0$ memory_percent41.24140739440918 disk.drbd1931.out0 disk.dm-0.in 2.47265625 disk.drbd1584.out0$ disk.nvme0n1.out175.54052734375' proc.memory.availableMB 38508.03125 disk.drbd1673.out0 disk.drbd1550.out0 disk.rbd0.in0 disk.drbd2078.out0 disk.drbd1289.in0 disk.sda2.in0r2- _wandbruntime3972  _runtime3972[q6 _step712 & lr-AdamW/pg10.0002306352968570043 & lr-AdamW/pg20.0002306352968570043 trainer/global_step8949 % _timestamp1.7857992599117246e+09 _runtime3972.734685647 & lr-AdamW/pg20.0002306352968570043 trainer/global_step8949 % _timestamp1.7857992599117246e+09 _runtime3972.734685647 _step712 & lr-AdamW/pg10.0002306352968570043zw2- _wandbruntime3972  _runtime3972z}'O XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated25639078 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 7874992022528 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 trainer/global_step8949 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed59459427 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 % _timestamp1.7857992599126701e+09 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed153647054562304 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed161521959155200 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed153647054562304 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed25638818 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated153647071601664 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated157684821628308 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated60877625 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 153978092 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed154009519694848 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed157684718785824 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed35238545 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated162097416054784 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 7874904592896 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed149829086941472 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated94518663 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed162097258379776 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated161522063624192 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated59459429 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed94509007 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 _step713 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed60877363 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 8087871456256 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed59459427 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated94518663 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed161521959155200 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated59459429 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated153647071601664 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current260 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated149829103980832 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed94509007 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed149829086941472 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated94518663 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed162097258379776 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated161522063624192 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated59459429 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed94509007 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 _step713 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed60877363 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 8087871456256 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed59459427 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated94518663 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed161521959155200 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated59459429 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated153647071601664 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current260 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated149829103980832 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed94509007 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed25638818 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated153647071601664 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated157684821628308 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated60877625 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 153978092 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed154009519694848 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed157684718785824 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed35238545 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated162097416054784 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656JP2- _wandbruntime3973  _runtime3973X 2[M S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed161540005537792 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated94529200 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed94519530 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 153995272 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime3973.120819961 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed25641688 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 trainer/global_step8949 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed35242482 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated59466072 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed154026726811648 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 7875871649280 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 153995272 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 8088774152704 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed157702336431366 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed149845827113792 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7856509317574 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed153664221325312 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated60884436 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 7875871649280 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated157702439273962 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed94519530 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated25641952 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current264 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 % _timestamp1.7857992602974782e+09 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 153985600 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed59466070 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated162115525868032 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 _step714 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated153664238364672 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated149845844153152 NADeviceStatsMonitor.on_train_batchaz_end/active.small_pool.allocated94529200 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed60884170 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated161540110013952 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 153985600 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed162115370297344 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed59466070 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 153995272 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 8088774152704 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed157702336431366 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed149845827113792 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7856509317574 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed153664221325312 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated60884436 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 7875871649280 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated157702439273962 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed94519530 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated25641952 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current264 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 % _timestamp1.7857992602974782e+09 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 153985600 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed59466070 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated162115525868032 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 _step714 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated153664238364672 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated149845844153152 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated94529200 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed60884170 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated161540110013952 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 153985600 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed162115370297344 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed59466070 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 153995272 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime3973.120819961 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed25641688 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 trainer/global_step8949 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed35242482 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated59466072 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed154026726811648 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 7875871649280 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed95037009 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed162424293441024 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated154869900438528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed95037009 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 154838244 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 7918900728320 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 8132926256640 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated154505409752064 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 trainer/global_step8999 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7918988157952 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed35435395 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated150666112596832 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated59791579 % _timestamp1.7857992821856446e+09 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed162424293441024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current236 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated163002957369344 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed59791577 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated25781360 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed154505392712704 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated35435397 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed25781126 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak118161iH>O MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed59791577 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated25781360 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed154505392712704 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated35435397 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed25781126 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed95037009 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed162424293441024 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated154869900438528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed95037009 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 154838244 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 7918900728320 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 8132926256640 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated154505409752064 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 trainer/global_step8999 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7918988157952 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed35435395 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated150666112596832 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated59791579 % _timestamp1.7857992821856446e+09 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed162424293441024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current236 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated163002957369344=2- _wandbruntime3995  _runtime3995aR@&M LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed35439332 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated154522576515072 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed95047532 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed154522559475712 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 7919867784704 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated35439334 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated61223532 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed61223275 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated154887107555328 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated25784198 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated162442444299776 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 7919867784704 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7900397781082 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 7919780347904 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 % _timestamp1.7857992825651507e+09 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated162442444299776 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated59798222 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 154855424 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed25783943 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed158583233510874 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed150682835729792 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated95057202 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 _step718&M QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated25784198 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated162442444299776 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 7919867784704 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7900397781082 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 7919780347904 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 % _timestamp1.7857992825651507e+09 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated162442444299776 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated59798222 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 154855424 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed25783943 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed158583233510874 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed150682835729792 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated95057202 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 _step718 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed35439332 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated154522576515072 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed95047532 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed154522559475712 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 7919867784704 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated35439334 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated61223532 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed61223275 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated154887107555328 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291k7q2- _wandbruntime3995  _runtime3995g _runtime3995.389690011 _step719 2loss/SRqqt1/assignment_loss1.1640999840633215 3loss/FBt2/assignment_loss0.0027619766116954866 2loss/SRqqt2/assignment_loss0.9255435661559234 3loss/FBt1/assignment_loss0.0001336778481086931 .loss/FRt1/detection_loss2.511110334169297 /loss/FRt2/detection_loss2.4883246324866652 1loss/SRqqt2/detection_loss0.9265245721660434 epoch0 0loss/FRt1/assignment_loss2.8764626525697254 /loss/FBt1/detection_loss1.2240737140382407 /loss/FBt2/detection_loss1.9871312908150933 %loss/total_loss18.28227115946582 trainer/global_step8999 /loss/FRt2/assignment_loss2.267765904047403 0loss/SRqqt1/detection_loss1.908338854494302 $ _timestamp1.785799282566845e+09.U  2loss/SRqqt1/assignment_loss1.1640999840633215 3loss/FBt2/assignment_loss0.0027619766116954866 2loss/SRqqt2/assignment_loss0.9255435661559234 3loss/FBt1/assignment_loss0.0001336778481086931 .loss/FRt1/detection_loss2.511110334169297 /loss/FRt2/detection_loss2.4883246324866652 1loss/SRqqt2/detection_loss0.9265245721660434 epoch0 0loss/FRt1/assignment_loss2.8764626525697254 /loss/FBt1/detection_loss1.2240737140382407 /loss/FBt2/detection_loss1.9871312908150933 %loss/total_loss18.28227115946582 trainer/global_step8999 /loss/FRt2/assignment_loss2.267765904047403 0loss/SRqqt1/detection_loss1.908338854494302 $ _timestamp1.785799282566845e+09 _runtime3995.389690011 _step719D`|U(: U gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts116.341' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent38.78033333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0[(:  U* proc.memory.availableMB38508.30078125 disk.drbd1550.out0 disk.drbd1226.in0 disk.drbd2111.out0 disk.drbd1965.out0 disk.drbd2118.out0 disk.nvme0n1.in 2.47265625! proc.memory.rssMB 11260.40625 disk.drbd2078.out0$ disk./.usageGB578.3484954833984 disk.drbd2111.in0 disk.rbd1.in0 disk.drbd1828.in0* proc.memory.percent17.182016372680664 disk.rbd0.out0 disk.sda1.in0 disk.drbd1285.in0! disk.dm-0.out176.15087890625 disk.drbd1817.out0 disk.drbd2041.in0 cpu13.796163563253481 network.sent 8.209365e+06 disk.sda1.out0 disk.drbd1931.out0 proc.cpu.threads29 disk.drbd1938.out0 disk.drbd1226.out0 disk.drbd2041.out0 disk.drbd1584.in0 network.recv515892 disk.drbd1289.in0 disk.drbd1289.out0 disk.drbd1817.in0 disk.drbd1673.out0 disk.drbd1931.in0 disk.drbd1552.out0 disk.drbd1550.in0 disk.drbd1510.in0 disk.sda2.in0 disk.sda2.out 109.80859375 disk.drbd1552.in0 disk.sda.out 109.80859375 disk.dm-0.in 2.47265625 disk.rbd1.out 15.931640625 disk.drbd1356.in0 disk.sda.in0 disk.drbd1938.in0 disk.drbd1285.out0$ disk.nvme0n1.out176.15087890625 disk.drbd1356.out0 disk.drbd1584.out0 disk.drbd2118.in0 disk.drbd1965.in0$ memory_percent41.24099612236023 disk.drbd1510.out0 disk.drbd2078.in0 disk.drbd1673.in0) disk./.usagePercent32.34427549376958 disk.rbd0.in0 disk.drbd1828.out0q2- _wandbruntime4017  _runtime4017>` _runtime4017.447150119 _step720 ' lr-AdamW/pg10.00023321251550553488 ' lr-AdamW/pg20.00023321251550553488 trainer/global_step9049 % _timestamp1.7857993046241934e+090 ' lr-AdamW/pg10.00023321251550553488 ' lr-AdamW/pg20.00023321251550553488 trainer/global_step9049 % _timestamp1.7857993046241934e+09 _runtime4017.447150119 _step7205u2- _wandbruntime4017  _runtime4017Jp/ O D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed159446512944840 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 trainer/global_step9049 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 7962896863744 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated155730256278528 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed60123727 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated95574667 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current230 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated61556901 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 8178101281792 % _timestamp1.7857993046251497e+09 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed155363730863104 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed95565011 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed163326627726848 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed155363730863104 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated163326732195840 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 155698396 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated163908490331648 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated95574667 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated25924654 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated60123729 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed61556669 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current232 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed163326627726848 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated35632247 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed25924424 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed35632245 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7962984293376 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 1327713282v'O MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated163326732195840 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated151503121212832 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated60123729 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed155730231374848 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 8178234053120 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 7943408771368 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated155363747902464 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed163908332656640 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 7943494574492 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 7962896863744 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed95565011 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 _runtime4017.448431178 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 155688738 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated155363747902464 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 7962984293376 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed159446512944840 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 trainer/global_step9049 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 7962896863744 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated155730256278528 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed60123727 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated95574667 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current230 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated61556901 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 8178101281792 % _timestamp1.7857993046251497e+09 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed155363730863104 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed95565011 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed163326627726848 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed155363730863104 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated163326732195840 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 155698396 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated163908490331648 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated95574667 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated25924654 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated60123729 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed61556669 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current232 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed163326627726848 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated35632247 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed25924424 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed35632245 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated60130372 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 155715576 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated163344778585600 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed155380897626112 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 7963776483328 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated25927554 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 8179012734464 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 7944372047826 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 7963863920128 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 7963863920128 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed155747438491648 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 % _timestamp1.7857993049992924e+09 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 155715576 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated95585204 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed155380897626112 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed60130370 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated35636184 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed163926451226112 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed151519844345792 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 155705904 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed60130370 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7944286244590 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated155380914665472 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated159464233432978 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated151519861385152 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated155747463395328 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated95585204 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 7963776483328 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current275 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 8179149692928 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed61563461 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated163926613088256 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed25927279 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated163344778585600 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed163344674109440 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed159464130590382 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed95575534 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated61563738 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 _step722 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 _runtime4017.822673886 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated155380914665472 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current277 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464|_yM A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated163344778585600 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed155380897626112 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 7963776483328 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated25927554 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 8179012734464 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 7944372047826 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 7963863920128 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 7963863920128 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed155747438491648 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 % _timestamp1.7857993049992924e+09 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 155715576 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated95585204 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed155380897626112 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed60130370 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated35636184 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed163926451226112 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed151519844345792 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 155705904 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed60130370 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7944286244590 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated155380914665472 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated159464233432978 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated151519861385152 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated155747463395328 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated95585204 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 796377T6483328 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current275 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 8179149692928 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed61563461 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated163926613088256 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed25927279 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated163344778585600 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed163344674109440 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed159464130590382 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed95575534 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated61563738 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 _step722 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 _runtime4017.822673886 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated155380914665472 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current277 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated60130372 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 155715576L@>2- _wandbruntime4017  _runtime4017 c+ /loss/FRt1/assignment_loss3.287163618632725 /loss/FRt1/detection_loss2.4188664016269503 1loss/SRqqt2/detection_loss1.0899143932845945 /loss/FRt2/assignment_loss2.197023619072778 1loss/SRqqt1/detection_loss1.7798312155431824 &loss/total_loss17.098857104169976 2loss/SRqqt1/assignment_loss0.9213822314417676 3loss/FBt1/assignment_loss0.0004329554862495474 1loss/FBt2/assignment_loss0.07425057149956515 /loss/FBt1/detection_loss0.7458204790018499 epoch0 % _timestamp1.7857993050009732e+09 2loss/SRqqt2/assignment_loss0.9642754293371812 .loss/FRt2/detection_loss2.363886883748429 /loss/FBt2/detection_loss1.2560093054947044 trainer/global_step9049 _runtime 4017.82380086 _step723#  _runtime 4017.82380086 _step723 /loss/FRt1/assignment_loss3.287163618632725 /loss/FRt1/detection_loss2.4188664016269503 1loss/SRqqt2/detection_loss1.0899143932845945 /loss/FRt2/assignment_loss2.197023619072778 1loss/SRqqt1/detection_loss1.7798312155431824 &loss/total_loss17.098857104169976 2loss/SRqqt1/assignment_loss0.9213822314417676 3loss/FBt1/assignment_loss0.0004329554862495474 1loss/FBt2/assignment_loss0.07425057149956515 /loss/FBt1/detection_loss0.7458204790018499 epoch0 % _timestamp1.7857993050009732e+09 2loss/SRqqt2/assignment_loss0.9642754293371812 .loss/FRt2/detection_loss2.363886883748429 /loss/FBt2/detection_loss1.2560093054947044 trainer/global_step9049vt(: U gpu.0.gpu24.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts125.337' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent41.778999999999996 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0ğgZ(:  U$ disk./.usageGB578.3484764099121 disk.drbd2041.in0 disk.drbd1289.out0 disk.drbd1938.out0 disk.sda.out 110.2265625 disk.drbd1356.in0 disk.drbd1285.in0$ memory_percent41.24071002006531! disk.dm-0.out176.45947265625 disk.drbd1931.out0* proc.memory.availableMB38508.48828125 network.recv517392 disk.drbd2111.in0 disk.rbd1.in0 cpu13.795320834339378 disk.drbd2118.out0$ disk.nvme0n1.out176.45947265625 disk.drbd1817.out0 disk.drbd1965.out0 disk.drbd1828.in0 disk.drbd1828.out0 disk.drbd1510.out0 disk.drbd1226.out0 disk.drbd2111.out0 disk.rbd1.out 16.1435546875 disk.drbd1552.in0 disk.drbd2078.in0 disk.drbd1550.out0 disk.rbd0.in0 disk.sda.in0 disk.sda1.out0 proc.cpu.threads29) disk./.usagePercent32.34427442708038 disk.drbd1931.in0 disk.drbd1550.in0! proc.memory.rssMB 11260.40625 disk.drbd1356.out0 disk.drbd1817.in0 disk.drbd1552.out0 disk.sda1.in0 disk.drbd1584.out0 disk.rbd0.out0 disk.drbd2118.in0 disk.nvme0n1.in 2.47265625 disk.sda2.in0 disk.drbd1673.in0 disk.drbd1289.in0 disk.drbd2078.out0 disk.drbd1584.in0 disk.sda2.out 110.2265625 disk.drbd2041.out0 disk.drbd1285.out0 disk.drbd1510.in0 network.sent 8.254381e+06 disk.dm-0.in 2.47265625 disk.drbd1226.in0* proc.memory.percent17.182016372680664 disk.drbd1673.out0 disk.drbd1965.in0 disk.drbd1938.in0՚(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts97.685' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent32.56166666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0?Z\(:  ςV disk.drbd1938.out0 disk.drbd1510.out0 disk.drbd1584.in0 disk.drbd2111.out0 disk.drbd2078.out0$ disk.nvme0n1.out176.70556640625 disk.drbd1828.in0 disk.drbd2041.out0 network.sent 8.299291e+06 disk.drbd1673.in0 disk.drbd1550.in0 disk.drbd1817.out0 disk.drbd1289.out0 disk.drbd1673.out0 disk.nvme0n1.in 2.47265625 disk.drbd1285.out0 disk.drbd1226.in0 disk.drbd1226.out0 disk.drbd2118.in0 disk.sda2.in0 disk.drbd2041.in0 disk.drbd1285.in0 disk.drbd1931.out0$ memory_percent41.24071002006531 disk.rbd0.out0 disk.dm-0.in 2.47265625! disk.dm-0.out176.70556640625 disk.drbd2111.in0 disk.rbd1.in0 disk.rbd1.out 16.1435546875 disk.drbd1356.in0 disk.drbd1931.in0 disk.drbd1828.out0 cpu13.79433083716427 disk.rbd0.in0 disk.sda2.out 110.81640625 disk.drbd1552.out0 disk.drbd1289.in0 proc.cpu.threads29 disk.drbd1938.in0 disk.drbd1510.in0 disk.drbd1965.out0 network.recv518817* disk./.usagePercent32.344326054837865 disk.sda1.out0 disk.drbd2078.in0 disk.drbd1356.out0 disk.drbd1552.in0* proc.memory.availableMB38508.48828125! proc.memory.rssMB 11260.40625 disk.sda.in0 disk.sda1.in0 disk.drbd2118.out0 disk.drbd1965.in0 disk.drbd1550.out0$ disk./.usageGB578.3493995666504 disk.sda.out 110.81640625 disk.drbd1584.out0 disk.drbd1817.in0* proc.memory.percent17.182016372680664(2- _wandbruntime4039  _runtime4039{# _step724 ' lr-AdamW/pg10.00023450112482980017 ' lr-AdamW/pg20.00023450112482980017 trainer/global_step9099 % _timestamp1.7857993265768688e+09 _runtime4039.399892202W _runtime4039.399892202 _step724 ' lr-AdamW/pg10.00023450112482980017 ' lr-AdamW/pg20.00023450112482980017 trainer/global_step9099 % _timestamp1.7857993265768688e+09׬v2- _wandbruntime4039  _runtime4039crx'O UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed164228962012672 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed26068238 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated156222086052864 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed60455877 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated152340129828832 _runtime 4039.40182519 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed35829095 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated156222086052864 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated61897571 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 156548890 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated164229066481664 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed164813894850048 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated96102669 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated26068474 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated60455879 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed96093013 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 8006892999168 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 156558548 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current236 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed156222069013504 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated60455879 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed96093013 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 7987297234876 _step725 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated96102669 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed60455877 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed156222069013504 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed164228962012672 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated160327512866832 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed152340112789472 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated156590612118528 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 7987383038000 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed156590587214848 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated35829097 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated164814048330752 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed160327410024348 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 trainer/global_step9099 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed156222069013504 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed164228962012672 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated160327512866832 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed152340112789472 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated156590612118528 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 7987383038000 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed156590587214848 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated35829097 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated164814048330752 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed160327410024348 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 trainer/global_step9099 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated152340129828832 _runtime 4039.40182519 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed35829095 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated156222086052864 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated61897571 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 156548890 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated164229066481664 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed164813894850048 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated96102669 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated26068474 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated60455879 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed96093013 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 8006892999168 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144,ml2- _wandbruntime4039  _runtime4039h+M TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 163959296 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 156566056 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 8007860055552 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 156575728 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 SDDeviceStatsMonitor.on_train_b_*"atch_end/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated156607819235328 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8007860055552 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated26071366 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 % _timestamp1.7857993269558825e+09 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated61904400 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current259 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed156239235776512 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed164832001664000 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 156575728 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed96103536 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed160345027669890 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated164832165623296 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 7988260511334 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed60462520 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8007772618752 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed96103536 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated164247112871424 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed152356852961792 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 156566056 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated60462522 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed60462520 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 trainer/global_step9099 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed61904141 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated35833034 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated160345130512486 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360&M C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed26071109 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 8224346387968 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 7988174708098 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed96103536 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed160345027669890 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated164832165623296 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 7988260511334 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed60462520 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8007772618752 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed96103536 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated164247112871424 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed152356852961792 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 156566056 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated60462522 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed60462520 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 trainer/global_step9099 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed61904141 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated35833034 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated160345130512486 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 163959296 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 156566056 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 8007860055552 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 156575728 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated156607819235328 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8007860055552 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated26071366 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 % _timestamp1.7857993269558825e+09 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated61904400 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current259 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed156239235776512 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed164832001664000 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 156575728 F(: ʧV gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts117.646' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent39.215333333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0-̮2- _wandbruntime4061  _runtime4061q ' lr-AdamW/pg20.00023578973415406548 trainer/global_step9149 % _timestamp1.7857993488481042e+09 _runtime 4061.67106445 _step728 ' lr-AdamW/pg10.000235789734154065487 ' lr-AdamW/pg20.00023578973415406548 trainer/global_step9149 % _timestamp1.7857993488481042e+09 _runtime 4061.67106445 _step728 ' lr-AdamW/pg10.00023578973415406548]2- _wandbruntime4061  _runtime4061Ll&O ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated157080424203264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed165131296298496 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 _runtime4061.672334197 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 8268635066880 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 8050889134592 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 126479872 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 % _timestamp1.7857993488490696e+09 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8031185698384 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _step729 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated60788029 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated96630671 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated96630671 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed165719451641856 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 8268508587008 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated62236951 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed26210754 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed153177121405472 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 8050976564224 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated26211004 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated60788029 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated165131400767488 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 trainer/global_step9149 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed62236699 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 8050976564224 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated157080424203264 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated165719603025408 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current252 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8031271501508 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8050889134592 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated153177138444832 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed161208307103856 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated165131400767488 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed96621015 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed157080407163904 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current250 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed60788027 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed36025945 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed165131296298496 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 _runtime4061.672334197 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 8268635066880 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 8050889134592 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 126479872 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 % _timestamp1.7857993488490696e+09 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8031185698384 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _step729 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated60788029 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated96630671 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated96630671 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed165719451641856 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 8268508587008 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated62236951 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed26210754 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed153177121405472 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 8050976564224 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated26211004 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated60788029 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated165131400767488 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 trainer/global_step9149 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed62236699 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 8050976564224 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated157080424203264 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated165719603025408 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current252 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8031271501508 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8050889134592 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated153177138444832 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed161208307103856 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated165131400767488 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed96621015 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed157080407163904 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current250 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed60788027 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed36025945 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated36029884 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated96641208 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated157097590966272 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8051856190976 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed157097573926912 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated60794672 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed96631538 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8051768754176 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 trainer/global_step9149 _step730 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current236 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 8269544605696 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed161225924749398 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated96641208 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated165149447157248 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed96631538 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated60794672 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed60794670 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed62243488 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current234 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8032063171606 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 157435880 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed157097573926912 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed165737562013184 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated153193878617152 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated157097590966272 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 157435880 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated157468175075328 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated161226027591994 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0y&M >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated36029884 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated96641208 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated157097590966272 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8051856190976 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed157097573926912 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated60794672 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed96631538 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8051768754176 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 trainer/global_step9149 _step730 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current236 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 8269544605696 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed161225924749398 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated96641208 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated165149447157248 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed96631538 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated60794672 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed60794670 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed62243488 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current234 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8032063171606 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 157435880 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed157097573926912 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed165737562013184 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated153193878617152 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated157097590966272 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 157435880 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated157468175075328 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated161226027591994 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0922- _wandbruntime4062  _runtime40624  2loss/SRqqt1/assignment_loss0.9673340259797194 /loss/FRt1/detection_loss2.4561189930329377 1loss/SRqqt1/detection_loss1.6094452380456707 1loss/SRqqt2/detection_loss0.9590554752155839 /loss/FRt2/assignment_loss2.347823718190193 2loss/SRqqt2/assignment_loss0.9416746153877485 /loss/FBt1/detection_loss0.6921683276264832 _step731 0loss/FRt1/assignment_loss2.6606099221898223 /loss/FRt2/detection_loss2.5008482018485667 /loss/FBt2/detection_loss2.0922327739759705 &loss/total_loss17.227311649436363 trainer/global_step9149 % _timestamp1.7857993492194796e+09 4loss/FBt1/assignment_loss1.5620507970628877e-07 4loss/FBt2/assignment_loss2.0173858563164407e-07 epoch0 _runtime4062.042308664y 2loss/SRqqt1/assignment_loss0.9673340259797194 /loss/FRt1/detection_loss2.4561189930329377 1loss/SRqqt1/detection_loss1.6094452380456707 1loss/SRqqt2/detection_loss0.9590554752155839 /loss/FRt2/assignment_loss2.347823718190193 2loss/SRqqt2/assignment_loss0.9416746153877485 /loss/FBt1/detection_loss0.6921683276264832 _step731 0loss/FRt1/assignment_loss2.6606099221898223 /loss/FRt2/detection_loss2.5008482018485667 /loss/FBt2/detection_loss2.0922327739759705 &loss/total_loss17.227311649436363 trainer/global_step9149 % _timestamp1.7857993492194796e+09 4loss/FBt1/assignment_loss1.5620507970628877e-07 4loss/FBt2/assignment_loss2.0173858563164407e-07 epoch0 _runtime4062.042308664e~(: U gpu.0.gpu28.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts130.404' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent43.468 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0W(:  V disk.drbd1828.out0! disk.dm-0.out177.33642578125 disk.drbd2118.out0 disk.nvme0n1.in 2.47265625 disk.drbd1965.in0 disk.d5Krbd1828.in0 disk.sda1.in0 disk.dm-0.in 2.47265625 disk.drbd1584.in0 disk.drbd1673.in0 disk.drbd1931.in0 proc.cpu.threads29 disk.drbd1550.out0 disk.drbd1938.out0 disk.drbd2118.in0 disk.rbd1.in0 disk.drbd2041.out0 disk.sda1.out0 disk.rbd0.in0 disk.drbd1285.in0& proc.memory.availableMB 38508.1875* disk./.usagePercent32.344328188216274 disk.rbd1.out 16.2060546875 disk.drbd1289.out0 disk.drbd1356.in0 disk.drbd1817.out0 disk.drbd1965.out0 cpu13.792587055969195 network.sent 8.347264e+06 disk.drbd1550.in0 disk.drbd1226.out0 disk.drbd1673.out0 disk.drbd1931.out0 disk.drbd1552.out0 disk.sda.out 111.63671875 disk.drbd2078.out0* proc.memory.percent17.182016372680664 network.recv521500 disk.drbd2041.in0 disk.drbd1584.out0$ disk.nvme0n1.out177.33642578125 disk.drbd1289.in0 disk.drbd1285.out0 disk.drbd1817.in0 disk.rbd0.out0 disk.drbd1510.in0 disk.drbd2078.in0# disk./.usageGB578.349437713623 disk.drbd1552.in0 disk.drbd1938.in0 disk.sda.in0 disk.drbd1226.in0 disk.drbd2111.out0 disk.drbd1356.out0 disk.sda2.in0 disk.sda2.out 111.63671875 disk.drbd2111.in0 disk.drbd1510.out0$ memory_percent41.24116897583008! proc.memory.rssMB 11260.40625\=(: U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts125.56' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent41.85333333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors05O(:  U) disk./.usagePercent32.34435442877069 disk.drbd1356.in0 disk.drbd2111.in0 disk.drbd1289.out0 disk.sda.out 112.1484375 disk.sda1.in0( proc.memory.availableMB 38508.390625 disk.nvme0n1.in 2.47265625 disk.drbd2041.in0 disk.drbd1938.in0% memory_percent41.240859031677246 disk.drbd2111.out0 disk.drbd2118.out0 disk.drbd1965.in0 disk.drbd1673.out0 disk.drbd2078.out0 disk.drbd1584.in0 disk.nvme0n1.out 177.7734375 disk.drbd1965.out0 disk.rbd1.in0 disk.drbd2118.in0 disk.drbd1356.out0 disk.drbd1289.in0 disk.rbd0.out0 network.sent 8.392167e+06 disk.drbd1584.out0 disk.rbd1.out 16.33203125 disk.drbd1931.out0 disk.drbd1828.out0 disk.drbd1550.in0 disk.drbd1285.out0 disk.drbd1510.out0 disk.dm-0.out 177.7734375 disk.rbd0.in0! proc.memory.rssMB 11260.40625 disk.sda2.in0 disk.drbd1817.in0 disk.drbd2041.out0 disk.drbd1550.out0 disk.drbd1226.out0 disk.sda2.out 112.1484375$ disk./.usageGB578.3499069213867 disk.drbd2078.in0 disk.drbd1552.out0 disk.sda.in0 cpu13.791770106468874 proc.cpu.threads29 disk.drbd1226.in0 disk.drbd1817.out0 disk.drbd1828.in0 disk.drbd1938.out0* proc.memory.percent17.182016372680664 disk.sda1.out0 disk.drbd1285.in0 disk.drbd1673.in0 disk.drbd1931.in0 disk.drbd1552.in0 network.recv522901 disk.drbd1510.in0 disk.dm-0.in 2.472656252- _wandbruntime4083  _runtime4083 % _timestamp1.7857993709140377e+09 _runtime4083.736997056 _step732 & lr-AdamW/pg10.0002370783434783308 & lr-AdamW/pg20.0002370783434783308 trainer/global_step9199 K _step732 & lr-AdamW/pg10.0002370783434783308 & lr-AdamW/pg20.0002370783434783308 trainer/global_step9199 % _timestamp1.7857993709140377e+09 _runtime4083.736997056B*n\2- _wandbruntime4083  _runtime4083Gh'O `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated157938762353664 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8075159965016 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed61120177 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 _step733 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed166033630584320 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated154014147060832 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 8313814422016 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8075074161892 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed26353651 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 8313681650688 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated26353909 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 158269194 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated158311323798528 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 % _timestamp1.7857993709150004e+09 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current260 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 trainer/global_step9199 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed36222795 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed166624980545536 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed158311298894848 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated97158673 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated61120179 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated166625138220544 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 _runtime4083.738188157 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed157938745314304 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated36222797 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated61120179 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed61120177 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated166033735053312Xl'O PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed61120177 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 _step733 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed166033630584320 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated154014147060832 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 8313814422016 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8075074161892 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed26353651 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 8313681650688 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated26353909 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 158269194 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated158311323798528 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 % _timestamp1.7857993709150004e+09 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current260 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 trainer/global_step9199 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed36222795 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed166624980545536 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed158311298894848 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated97158673 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated61120179 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated166625138220544 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 _runtime4083.738188157 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed157938745314304 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated36222797 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated61120179 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed61120177 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated166033735053312 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated157938762353664 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8075159965016 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291ni2- _wandbruntime4084  _runtime4084DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed97159540 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed157955912077312 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _step734 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed36226732 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated157955929116672 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed166643091897856 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 158296032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 8314714456064 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated166643245371392 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed162106821828906 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated61126822 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8095852326400 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 158286360 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8095764889600 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated26356711 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 trainer/global_step9199 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed157955912077312 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated166051781443072 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 158296032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated61126822 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated97169210 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated97169210 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8076037438350 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated154030887233152 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8075951635114 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 8095852326400 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed61126820 UHDeviceSt#ҝatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed154030870193792 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated166051781443072 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed158328506011648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 _runtime4084.107096052 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 158286360 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed62583222 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated36226734 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current221 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed61126820 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032O|&M H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 158296032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 8314714456064 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated166643245371392 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed162106821828906 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated61126822 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8095852326400 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 158286360 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8095764889600 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated26356711 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 trainer/global_step9199 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed157955912077312 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated166051781443072 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 158296032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated61126822 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated97169210 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated97169210 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8076037438350 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated154030887233152 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8075951635114 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 8095852326400 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed61126820 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed154030870193792 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated166051781443072 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed158328506011648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 _runtime4084.107096052 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 158286360 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed62583222 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated36226734 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current221 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed61126820 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current223 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed166051676966912 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed97159540 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed157955912077312 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _step734 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed36226732 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated157955929116672 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed166643091897856 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121|2- _wandbruntime4084  _runtime4084O" /loss/FRt1/assignment_loss2.866507120651774 2loss/SRqqt2/assignment_loss1.0925480354021513 4loss/FBt2/assignment_loss1.4512277533887517e-06 1loss/SRqqt2/detection_loss0.9743360708257364 2loss/SRqqt1/assignment_loss0.8108312417309553 3loss/FBt1/assignment_loss6.468973948450049e-05 /loss/FRt1/detection_loss2.3740167266661576 .loss/FBt1/detection_loss1.217085418037393 epoch0 _step735 -loss/FRt2/detection_loss2.32252450488946 /loss/FBt2/detection_loss1.2391809227511934 $loss/total_loss16.5385802713724 0loss/FRt2/assignment_loss1.8383478010402006 1loss/SRqqt1/detection_loss1.8031362884101414 trainer/global_step9199 % _timestamp1.7857993712849178e+09 _runtime4084.1077561643  trainer/global_step9199 % _timestamp1.7857993712849178e+09 _runtime4084.107756164 /loss/FRt1/assignment_loss2.866507120651774 2loss/SRqqt2/assignment_loss1.0925480354021513 4loss/FBt2/assignment_loss1.4512277533887517e-06 1loss/SRqqt2/detection_loss0.9743360708257364 2loss/SRqqt1/assignment_loss0.8108312417309553 3loss/FBt1/assignment_loss6.468973948450049e-05 /loss/FRt1/detection_loss2.3740167266661576 .loss/FBt1/detection_loss1.217085418037393 epoch0 _step735 -loss/FRt2/detection_loss2.32252450488946 /loss/FBt2/detection_loss1.2391809227511934 $loss/total_loss16.5385802713724 0loss/FRt2/assignment_loss1.8383478010402006 1loss/SRqqt1/detection_loss1.8031362884101414`d(: ǖU gpu.0.gpu11.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts128.438' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.81266666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0‡;S(:  U disk.drbd1285.in0 disk.drbd1584.in0 disk.drbd2118.out0 disk.drbd1552.out0 disk.drbd1965.in0 disk.drbd1931.in0 disk.drbd1828.out0 disk.sda.out 112.69140625 network.sent 8.395178e+06 disk.sda2.out 112.69140625! proc.memory.rssMB 11260.40625 disk.drbd1510.in0 disk.drbd1817.out0 disk.drbd1828.in0 disk.drbd1817.in0 disk.drbd1673.out0 disk.drbd1285.out0 disk.drbd1226.in0 disk.drbd2078.in0 disk.drbd1673.in0 disk.drbd1550.out0 disk.drbd1938.in0 disk.sda2.in0 disk.drbd1226.out0 disk.drbd2111.out0 disk.rbd1.out 16.33203125 disk.nvme0n1.in 2.47265625 disk.drbd1356.out0 disk.rbd1.in0 disk.drbd2041.out0 disk.dm-0.in 2.47265625 disk.dm-0.out 177.94140625 disk.drbd2078.out0 disk.drbd2118.in0! disk.nvme0n1.out 177.94140625 disk.drbd1931.out0 disk.drbd1550.in0 disk.drbd1289.out0 disk.drbd2041.in0 disk.rbd0.in0% memory_percent41.240906715393066 proc.cpu.threads29 network.recv523940) disk./.usagePercent32.34435122870308$ disk./.usageGB578.3498497009277 disk.sda1.out0 disk.drbd1584.out0 disk.drbd1552.in0 disk.drbd1938.out0* proc.memory.percent17.182016372680664 disk.sda1.in0 disk.drbd1356.in0 disk.drbd1510.out0 disk.drbd2111.in0 cpu13.790960321638746 disk.drbd1965.out0 disk.drbd1289.in0 disk.rbd0.out0 disk.sda.in0( proc.memory.availableMB 38508.359375A!2- _wandbruntime4105  _runtime4105G _step736 ' lr-AdamW/pg10.00023836695280259607 ' lr-AdamW/pg20.00023836695280259607 trainer/global_step9249 $ _timestamp1.785799392802805e+09 _runtime4105.625776725 ' lr-AdamW/pg20.00023836695280259607 trainer/global_step9249 $ _timestamp1.785799392802805e+09 _runtime4105.625776725 _step736 ' lr-AdamW/pg10.00023836695280259607:2- _wandbruntime4105  _runtime4105i_c'O QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 8138881405440 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 8138968835072 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated61452329 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed158797083464704 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 159139004 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed166935964870144 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated158797100504064 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated26497321 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 _step737 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current250 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8119048428524 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 trainer/global_step9249 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated166936069339136 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed154851138637472 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 8138968835072 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated36419647 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated61452329 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 8358880664064 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8118962625400 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 8359017629696 % _timestamp1.7857993928037202e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated62916968 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed97677019 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 159129346 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated158797100504064 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8138881405440 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated154851155676832 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed36419645 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed97677019 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed167530535398912 _runtime4105.630671172 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated97686675 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current252 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed26497071 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed158797083464704 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 159139004 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed166935964870144 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated158797100504064 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated26497321 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 _step737 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current250 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8119048428524 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 trainer/global_step9249 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated166936069339136 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed154851138637472 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 8138968835072 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated36419647 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated61452329 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_byY۞ tes.small_pool.freed 8358880664064 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8118962625400 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 8359017629696 % _timestamp1.7857993928037202e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated62916968 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed97677019 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 159129346 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated158797100504064 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8138881405440 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated154851155676832 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed36419645 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed97677019 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed167530535398912 _runtime4105.630671172 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291Φ2- _wandbruntime4106  _runtime4106[u&M K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed62923547 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed159188861851648 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 159156184 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed97687542 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed166954011252736 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 8139761025024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 % _timestamp1.7857993931814067e+09 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed36423582 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed61458970 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed26499965 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed154867878809792 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8139848461824 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 8359918779904 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated154867895849152 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8119925901858 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 _runtime4106.004758127 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated166954115728896 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated97697212 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated158814267267072 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed97687542 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8139761025024 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed158814250227712 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 _step738 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed158814250227712 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed167548645770240 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated162987821751010 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed162987718908414 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated159188886755328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 trainer/global_step9249 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 159156184 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated61458972 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 159146512 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 8359783918592 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated61458972 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current261 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated62923810 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8119840098622 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 159146512 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed61458970 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated166954115728896 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated154867895849152 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8119925901858 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 _runtime4106.004758127 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated166954115728896 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated97697212 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated158814267267072 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed97687542 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8139761025024 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed158814250227712 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 _step738 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed158814250227712 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed167548645770240 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated162987821751010 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed162987718908414 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated159188886755328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 trainer/global_step9249 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 159156184 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated61458972 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 159146512 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 8359783918592 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated61458972 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current261 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated62923810 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8119840098622 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 159146512 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed61458970 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated166954115728896 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed62923547 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed159188861851648 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 159156184 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed97687542 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed166954011252736 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 8139761025024 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 % _timestamp1.7857993931814067e+09 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed36423582 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed61458970 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed26499965 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed154867878809792 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8139848461824 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 8359918779904 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 EDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed167838299155968 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed163850998342380 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated98214677 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed26640696 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated36616497 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed167838299155968 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 159999156 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8182877540864 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed98205021 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 8182964970496 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed159655421615104 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 159989498 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated61784479 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current250 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated155688164292832 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed168436074092032 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated160032035478528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated163851101184864 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed155688147253472 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated167838403624960 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 8182877540864 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated168436231767040 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed167838299155968 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed163850998342380 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated98214677 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed26640696 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated36616497 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed167838299155968 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 159999156 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8182877540864 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed98205021 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 8182964970496 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed159655421615104 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 159989498 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated61784479 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current250 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated155688164292832 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed168436074092032 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated160032035478528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated163851101184864 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed155688147253472 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated167838403624960 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200l2- _wandbruntime4128  _runtime4128"G3&M J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed36620432 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated155704904465152 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated168454340621824 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated159672605417472 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8183844597248 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed167856345538560 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed98215544 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed163868615987922 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 160006664 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 160016336 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed159672588378112 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8163728562130 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current250 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current248 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed26643493 _step742 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated167856450014720 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated61791122 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 % _timestamp1.7857994154431932e+09 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed63263925 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed61791120 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated159672605417472 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated167856450014720 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed159672588378112 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 8183844597248 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated63264175 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed61791120 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 8404967359488 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed168454185051136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 160006664xXM ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated98225214 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated36620434 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed160049217691648 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed155704887425792 trainer/global_step9299 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 160016336 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed159672588378112 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8163728562130 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current250 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current248 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed26643493 _step742 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated167856450014720 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated61791122 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 % _timestamp1.7857994154431932e+09 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed63263925 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed61791120 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated159672605417472 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated167856450014720 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed1596p72588378112 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 8183844597248 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated63264175 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed61791120 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 8404967359488 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed168454185051136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 160006664 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed36620432 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated155704904465152 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated168454340621824 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated159672605417472 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8183844597248 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed167856345538560 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed98215544 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed163868615987922 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 160006664 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 160016336 EDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed168740633441792 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated160513776804864 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 8226873676288 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8206825355540 _step745 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed156525155869472 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed26782151 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 160859308 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated156525172908832 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated63595741 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 8226961105920 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8206739552416 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed36813345 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed63595496 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8226873676288 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed168740633441792 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated36813347 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 _runtime4149.778296037 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 160849650 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed164731895421888 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 8226961105920 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated98742679 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated62116629 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed160513759765504 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed169341609089024 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed160892366414848 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated98742679 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed98733023 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated168740737910784 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed168740633441792 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated160513776804864 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 8226873676288 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8206825355540 _step745 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed156525155869472 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed26782151 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 160859308 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated156525172908832 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated63595741 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 8226961105920 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8206739552416 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed36813345 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed63595496 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8226873676288 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed168740633441792 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated36813347 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 _runtime4149.778296037 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 160849650 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed164731895421888 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 8226961105920 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated98742679 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated62116629 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed160513759765504 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed169341609089024 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed160892366414848 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak002- _wandbruntime4150  _runtime4150a  M TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated160909598435328 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed160909573531648 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated62123272 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 8450279944192 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed160530926528512 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated36817284 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 160876488 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 trainer/global_step9349 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 160876488 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8227840732672 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed62123270 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current256 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 160866816 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated63602464 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8207702828874 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed26784926 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed164749513067430 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8207617025638 H?DeviceStatsMoniTtor.on_train_batch_end/active.small_pool.current9670 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated156541913081152 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed98743546 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated168758784300544 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed160530926528512 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated160530943567872 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated98753216 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated168758784300544 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed169359716517376 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed36817282 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed63602208 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed62123270 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated160530943567872 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 8450142985728 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current254 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed98743546 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 _runtime4150.154143623 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated98753216 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated168758784300544 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed169359716517376 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed36817282 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed63602208 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed62123270 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated160530943567872 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 8450142985728 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current254 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed98743546 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 _runtime4150.154143623 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated62123272 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 8450279944192 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed160530926528512 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated36817284 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 160876488 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 trainer/global_step9349 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 160876488 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8227840732672 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed62123270 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current256 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 160866816 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated63602464 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8207702828874 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed26784926 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed164749513067430 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8207617025638 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated156541913081152 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0CѢ2- _wandbruntime4150  _runtime4150  2loss/SRqqt1/assignment_loss0.8629991380541131 /loss/FRt2/detection_loss2.1996061205863953 /loss/FRt1/detection_loss2.7526215659944633 1loss/SRqqt2/detection_loss1.0298235440844452 &loss/total_loss18.074662609627385 trainer/global_step9349 % _timestamp1.7857994373324459e+09 _runtime4150.155273538 3loss/FBt1/assignment_loss9.169944256917425e-09 3loss/FBt2/assignment_loss9.174889715839956e-05 1loss/SRqqt1/detection_loss1.7696782334296162 .loss/FBt1/detection_loss1.148296098332279 epoch0 0loss/FRt2/assignment_loss2.3597256521602255 2loss/SRqqt2/assignment_loss1.0231053503593066 .loss/FBt2/detection_loss2.006445205046071 _step747 0loss/FRt1/assignment_loss2.9222699435133683z. 0loss/FRt1/assignment_loss2.9222699435133683 2loss/SRqqt1/assignment_loss0.8629991380541131 /loss/FRt2/detection_loss2.1996061205863953 /loss/FRt1/detection_loss2.7526215659944633 1loss/SRqqt2/detection_loss1.0298235440844452 &loss/total_loss18.074662609627385 trainer/global_step9349 % _timestamp1.7857994373324459e+09 _runtime4150.155273538 3loss/FBt1/assignment_loss9.169944256917425e-09 3loss/FBt2/assignment_loss9.174889715839956e-05 1loss/SRqqt1/detection_loss1.7696782334296162 .loss/FBt1/detection_loss1.148296098332279 epoch0 0loss/FRt2/assignment_loss2.3597256521602255 2loss/SRqqt2/assignment_loss1.0231053503593066 .loss/FBt2/detection_loss2.006445205046071 _step747Iw(: U gpu.0.gpu38.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts98.584' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent32.861333333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0hQY(:  U disk.sda1.out0 disk.drbd2111.out0( proc.memory.availableMB 38507.609375 disk.drbd1356.out0 disk.drbd1552.in0 cpu13.78759507596957% memory_percent41.242051124572754 disk.drbd1584.out0! disk.dm-0.out179.01513671875 disk.drbd1931.in0 disk.drbd1289.in0$ disk.nvme0n1.out179.01513671875 disk.drbd2041.in0 disk.sda.out 114.67578125 disk.sda2.out 114.67578125) disk./.usagePercent32.34438280270353 disk.rbd0.out0 disk.sda1.in0 disk.drbd1226.out0 disk.drbd2078.in0 disk.rbd1.out 16.5205078125 disk.drbd2041.out0 disk.drbd1552.out0 disk.drbd1226.in0 disk.drbd1817.out0 disk.drbd1550.in0! proc.memory.rssMB 11260.40625 disk.sda2.in0 proc.cpu.threads29 disk.drbd1289.out0 disk.drbd1828.out0 disk.drbd2118.in0 network.recv529795 disk.rbd0.in0 disk.sda.in0 disk.drbd1285.out0 disk.drbd2078.out0 disk.drbd1356.in0 disk.drbd1673.in0 disk.drbd1510.in0 disk.drbd1510.out0 disk.drbd2111.in0 disk.drbd1938.in0 disk.drbd1965.out0# disk./.usageGB578.350414276123 disk.drbd1584.in0 disk.drbd2118.out0 disk.rbd1.in0 disk.drbd1965.in0 disk.drbd1828.in0 disk.drbd1550.out0 network.sent 8.533387e+06 disk.dm-0.in 2.47265625 disk.drbd1817.in0 disk.drbd1938.out0 disk.drbd1285.in0 disk.nvme0n1.in 2.47265625 disk.drbd1673.out0 disk.drbd1931.out0* proc.memory.percent17.182016372680664(: ūU gpu.0.gpu32.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts123.327' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent41.109 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0jY(:  V disk.drbd1550.in0 disk.drbd1356.out0 disk.rbd1.in0) proc.memory.availableMB 38507.3671875 cpu13.786721631408014 disk.drbd1965.in0 disk.drbd1828.out0 disk.drbd2041.in0 disk.drbd2041.out0 disk.drbd1510.in0 disk.dm-0.in 2.47265625 disk.sda2.out 115.0078125 proc.cpu.threads29 disk.drbd1552.out0 disk.drbd1226.in0 disk.drbd1552.in0 disk.drbd1828.in0 disk.drbd1965.out0$ memory_percent41.24242067337036! proc.memory.rssMB 11260.40625 disk.drbd1931.out0 disk.sda.out 115.0078125 disk.drbd1289.out0 disk.drbd1938.out0 disk.drbd1584.in0 disk.drbd2111.out0 disk.drbd1817.out0) disk./.usagePercent32.34438493608193 disk.drbd1673.out0 disk.rbd0.out0 network.recv531263 disk.drbd1285.in0 disk.drbd2118.in0 disk.drbd1356.in0 disk.drbd1931.in0 disk.drbd1817.in0 disk.drbd1673.in0 disk.drbd1938.in0 disk.sda1.in0 disk.sda1.out0 disk.drbd1510.out0 disk.drbd2118.out0$ disk./.usageGB578.3504524230957 disk.drbd1550.out0 disk.drbd1289.in0 disk.drbd2078.in0 disk.drbd1584.out0 disk.nvme0n1.in 2.47265625 disk.sda.in0 disk.drbd1285.out0 disk.drbd2078.out0 disk.rbd1.out 16.5830078125* proc.memory.percent17.182016372680664 network.sent 8.578293e+06 disk.rbd0.in0! disk.dm-0.out179.38134765625 disk.drbd1226.out0 disk.drbd2111.in0$ disk.nvme0n1.out179.38134765625 disk.sda2.in0N2- _wandbruntime4171  _runtime4171' ' lr-AdamW/pg20.00024223278077539203 trainer/global_step9399 % _timestamp1.7857994589979384e+09 _runtime4171.822723463 _step748 ' lr-AdamW/pg10.00024223278077539203/Ep ' lr-AdamW/pg20.00024223278077539203 trainer/global_step9399 % _timestamp1.7857994589979384e+09 _runtime4171.822723463 _step748 ' lr-AdamW/pg10.00024223278077539203:E2- _wandbruntime4171  _runtime4171]9'O A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated99270681 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated161372114955264 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed161372097915904 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated165612895343880 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 8270957241344 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated26924312 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current257 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated170247323786240 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 161709802 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed169642967727616 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed63934250 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated157362181524832 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 _runtime4171.822814617 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8270869811712 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current259 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed157362164485472 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed37010195 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 trainer/global_step9399 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed62448777 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated99270681 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated62448779 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed169642967727616 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 8270957241344 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed99261025 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated63934509 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated62448779 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed26924055 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed170247166111232 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed161752722254848 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed165612792501396 _step749 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 8494576627712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 161719460 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 8494443856384 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8250713819048 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated161372114955264 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated169643072196608 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 % _timestamp1.7857994589986293e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 8270869811712 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated161752747158528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed63934250 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated157362181524832 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 _runtime4171.822814617 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8270869811712 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current259 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed157362164485472 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed37010195 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 trainer/global_step9399 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed62448777 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated99270681 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated62448779 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed169642967727616 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 8270957241344 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed99261025 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated63934509 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated62448779 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed26924055 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed170247166111232 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed161752722254848 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed165612792501396 _step749 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 8494576627712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 161719460 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 8494443856384 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8250713819048 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated161372114955264 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated169643072196608 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 % _timestamp1.7857994589986293e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 8270869811712 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated161752747158528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 JDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated169661118586368 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 126472704 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 8271836868096 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 161736640 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed63941024 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed62455420 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated63941270 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated161769954275328 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed161769929371648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 161726968 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed169661014110208 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated62455422 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed37014132 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated26927136 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated99281218 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated157378921697152 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed26926892 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 trainer/global_step9399 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 161726968 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 8271749431296 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 % _timestamp1.7857994593815808e+09 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 161736640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current21Ւ&M LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current246 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated170265433075200 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8251505489146 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed99271548 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 _step750 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed161389264678912 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 151376384 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8271836868096 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated169661118586368 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 126472704 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 8271836868096 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 161736640 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed63941024 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed62455420 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated63941270 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated161769954275328 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed161769929371648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 161726968 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed169661014110208 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated62455422 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed37014132 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated26927136 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated99281218 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated157378921697152 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed26926892 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 trainer/global_step9399 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 161726968 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 8271749431296 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 % _timestamp1.7857994593815808e+09 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 161736640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2ay/2- _wandbruntime4172  _runtime4172 1loss/SRqqt2/assignment_loss0.895996995917004 3loss/FBt2/assignment_loss0.0023932144703926804 1loss/SRqqt1/detection_loss1.7400300222926024 &loss/total_loss16.541230669426128 2loss/SRqqt1/assignment_loss0.6302050060433585 4loss/FBt1/assignment_loss1.7881358293049165e-07 /loss/FRt2/detection_loss2.2640212882154933 0loss/SRqqt2/detection_loss1.136700406916336 .loss/FRt1/detection_loss2.715431163223778 epoch0 trainer/global_step9399 _step751 /loss/FRt1/assignment_loss2.689502476918988 0loss/FRt2/assignment_loss2.0882564342270293 .loss/FBt1/detection_loss1.095064899164506 /loss/FBt2/detection_loss1.2836285832230574 % _timestamp1.7857994593833346e+09 _runtime4172.206152305ղ  /loss/FRt1/assignment_loss2.689502476918988 0loss/FRt2/assignment_loss2.0882564342270293 .loss/FBt1/detection_loss1.095064899164506 /loss/FBt2/detection_loss1.2836285832230574 % _timestamp1.7857994593833346e+09 _runtime4172.206152305 1loss/SRqqt2/assignment_loss0.895996995917004 3loss/FBt2/assignment_loss0.0023932144703926804 1loss/SRqqt1/detection_loss1.7400300222926024 &loss/total_loss16.541230669426128 2loss/SRqqt1/assignment_loss0.6302050060433585 4loss/FBt1/assignment_loss1.7881358293049165e-07 /loss/FRt2/detection_loss2.2640212882154933 0loss/SRqqt2/detection_loss1.136700406916336 .loss/FRt1/detection_loss2.715431163223778 epoch0 trainer/global_step9399 _step751`r(: U gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts96.052' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent32.01733333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0e(:  U disk.drbd1673.in0 cpu13.786006328527426 disk.dm-0.out 180.267578125 disk.sda2.in0 disk.drbd1931.out0 disk.drbd2111.out0 disk.drbd1817.in0 disk.drbd1550.out0 disk.rbd0.in0! proc.memory.rssMB 11260.40625 disk.sda2.out 115.3515625 disk.rbd1.out 16.6689453125 disk.drbd1965.in0 disk.drbd1289.in0 disk.drbd1938.out0 disk.rbd0.out0 disk.rbd1.in0 disk.drbd1550.i&un0 network.recv532334 disk.drbd2078.in0 disk.drbd2111.in0 disk.drbd1965.out0 disk.drbd2041.in0 disk.drbd1584.in0 disk.drbd2041.out0 disk.drbd1828.out0 disk.sda1.out0# disk./.usageGB578.350471496582" disk.nvme0n1.out 180.267578125 disk.sda.in0 disk.sda.out 115.3515625 disk.drbd1285.out0 disk.drbd1510.out0 disk.dm-0.in 2.47265625 disk.drbd1356.in0 disk.drbd1931.in0 disk.drbd1552.in0 disk.drbd1938.in0 disk.drbd1356.out0 disk.drbd1673.out0 disk.drbd1828.in0 disk.drbd1552.out0 disk.drbd1289.out0$ memory_percent41.24246835708618 network.sent 8.581342e+06 disk.drbd1510.in0 disk.drbd2078.out0 disk.drbd1584.out0) proc.memory.availableMB 38507.3359375* disk./.usagePercent32.344386002771145 disk.drbd2118.in0 disk.drbd2118.out0 disk.nvme0n1.in 2.47265625 disk.sda1.in0* proc.memory.percent17.182016372680664 proc.cpu.threads29 disk.drbd1226.in0 disk.drbd1817.out0 disk.drbd1285.in0 disk.drbd1226.out0ʨ_2- _wandbruntime4193  _runtime4193( _runtime4193.739008205 _step752 ' lr-AdamW/pg10.00024352139009965732 ' lr-AdamW/pg20.00024352139009965732 trainer/global_step9449 % _timestamp1.7857994809160268e+09wL % _timestamp1.7857994809160268e+09 _runtime4193.739008205 _step752 ' lr-AdamW/pg10.00024352139009965732 ' lr-AdamW/pg20.00024352139009965732 trainer/global_step9449>!2- _wandbruntime4193  _runtime4193Qyh'O N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated99798683 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 8314953376768 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated162613102998528 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated158199190140832 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated62780929 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated62780929 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current257 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed64274554 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed62780927 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated162230453105664 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 8539592384512 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 _runtime 4193.74368618 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8314865947136 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed170545302013440 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 162569954 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated170545406482432 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated37207047 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated171152828154368 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed162613078094848 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed158199173101472 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 8314953376768 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated162230453105664 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 trainer/global_step9449 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8294516479432 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed171152670479360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 162579612 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 _step753 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed62780927 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated170545406482432 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated99798683 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 8314865947136 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated64274813 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed37207045 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed99789027 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8294602282556]vu'O YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed162230436066304 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated27067766 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 trainer/global_step9449 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8294516479432 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed171152670479360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 162579612 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 _step753 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed62780927 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated170545406482432 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated99798683 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 8314865947136 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated64274813 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed37207045 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed99789027 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8294602282556 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated99798683 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 8314953376768 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated162613102998528 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated158199190140832 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated62780929 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated62780929 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current257 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed64274554 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed62780927 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated162230453105664 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 8539592384512 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 _runtime 4193.74368618 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8314865947136 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed170545302013440 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 162569954 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated170545406482432 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated37207047 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated171152828154368 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed162613078094848 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed158199173101472 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 8314953376768 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated162230453105664 J6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 8315833003520 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 162596792 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 162587120 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated162630310115328 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated170563452872192 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed99799550 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 % _timestamp1.7857994812855399e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated62787572 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated99809220 trainer/global_step9449 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 162596792 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8315833003520 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed166511307226446 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 162587120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated162247619868672 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 8315745566720 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed62787570 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated64281582 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8295393952654 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 _runtime4194.108876615 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed64281322 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8315745566720 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed62787570 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated37210984 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated166511410069042 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated162247619868672 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current260 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated27070598 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed162630285211648 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated158215930313152 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current258 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed27070340 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 8540492458496 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8295479755890 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed162247602829312 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed171170777670144 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed158215913273792 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated170563452872192 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed99799550 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed162247602829312 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed37210982 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 1456512008-M NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated99809220 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated27070598 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed162630285211648 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated158215930313152 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current258 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed27070340 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 8540492458496 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8295479755890 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed162247602829312 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed171170777670144 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed158215913273792 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated170563452872192 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed99799550 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed162247602829312 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed37210982 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 _step754 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 8315833003520 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 162596792 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 162587120 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated162630310115328 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated170563452872192 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed99799550 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 % _timestamp1.7857994812855399e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated62787572 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated99809220 trainer/global_step9449 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 162596792 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8315833003520 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed166511307226446 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 162587120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated162247619868672 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 8315745566720 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed62787570 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated64281582 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8295393952654 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 _runtime4194.108876615 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed64281322 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8315745566720 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed62787570 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated37210984 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated166511410069042 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated162247619868672 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current260 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 BJ (: W gpu.0.gpu0.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts95.846' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.948666666666668 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0f;(: U gpu.0.gpu14.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts126.877' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.29233333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0Z5}X(:  V disk.drbd1673.out0 disk.rbd0.in0 disk.drbd1226.in0 disk.drbd1226.out0 disk.sda2.out 116.265625 disk.drbd1552.out0 disk.sda1.in0 disk.drbd1285.out0 disk.drbd1965.in0! proc.memory.rssMB 11260.40625 proc.cpu.threads29 disk.drbd2078.in0 disk.drbd1673.in0 disk.drbd2041.in0 disk.drbd1828.out0 disk.drbd1938.out0 disk.drbd1584.out0 network.sent 8.671285e+06 disk.drbd1931.out0 disk.drbd1550.out0 disk.dm-0.in 2.47265625* proc.memory.percent17.182016372680664 disk.rbd1.out 16.8251953125 disk.drbd1289.in0 disk.drbd1584.in0 disk.drbd2118.out0 disk.nvme0n1.in 2.47265625 disk.drbd1356.in0 disk.drbd2111.in0 disk.drbd1817.out0 disk.drbd1938.in0 disk.drbd1510.in0 disk.rbd1.in0$ memory_percent41.24346375465393* proc.memory.availableMB38506.68359375 disk.drbd2041.out0 disk.sda1.out0 disk.drbd2118.in0 disk.drbd1356.out0 disk.drbd1965.out0 cpu13.784409057053702 disk.drbd1931.in0 disk.sda.out 116.265625$ disk.nvme0n1.out181.92822265625 network.recv535299 disk.drbd1550.in0 disk.drbd1510.out0 disk.drbd2078.out0 disk.sda2.in0 disk.drbd2111.out0 disk.drbd1817.in0 disk.drbd1552.in0 disk.drbd1828.in0 disk.rbd0.out0 disk.sda.in0) disk./.usagePercent32.34438493608193 disk.drbd1285.in0 disk.drbd1289.out0! disk.dm-0.out181.92822265625$ disk./.usageGB578.3504524230957RL2- _wandbruntime4215  _runtime4215> & lr-AdamW/pg10.0002448099994239226 & lr-AdamW/pg20.0002448099994239226 trainer/global_step9499 $ _timestamp1.785799503016081e+09 _runtime4215.839032268 _step756 & lr-AdamW/pg10.0002448099994239226 & lr-AdamW/pg20.0002448099994239226 trainer/global_step9499 $ _timestamp1.785799503016081e+09 _runtime4215.839032268 _step7562- _wandbruntime4215  _runtime4215&'O GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed167374586660412 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 8358862082560 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated37403897 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated64613304 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed163088774216704 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 100326685 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed27209138 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated167374689502896 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 _step757 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated63113079 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated163473458838528 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 163439764 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed64613033 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed171447636299264 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8338490746064 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8358862082560 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated171447740768256 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed171447636299264 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed163088774216704 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 8358949512192 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated63113079 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 100317029 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed37403895 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed63113077 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8338404942940 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 8358949512192 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 8584777066496 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed159036181717472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 trainer/global_step9499n_O WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated171447740768256 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed171447636299264 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed163088774216704 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 8358949512192 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated63113079 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 100317029 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed37403895 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed63113077 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8338404942940 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 8358949512192 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 8584777066496 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed159036181717472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 trainer/global_step9499 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed167374586660412 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 8358862082560 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated37403897 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated64613304 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed163088774216704 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 QR]| CDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 100326685 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed27209138 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated167374689502896 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 _step757 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated63113079 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated163473458838528 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 163439764 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed64613033 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed171447636299264 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8338490746064 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8358862082560 60DeviceStatsMonitor.on_train_batch_start/num_ooms0[)2- _wandbruntime4216  _runtime4216&M&M ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated159052938929152 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 163456944 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 8359741702144 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 163456944 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed172076323017216 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 8585814729728 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 8585681965568 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 _step758 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed27211998 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated27212241 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated172076480685056 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed64619830 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated163105958019072 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 8359829138944 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated167392307148550 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated64620075 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed159052921889792 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8359829138944 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed163490641051648 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated163490665955328 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed63119720 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated171465787158016 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated171465787158016 _runtime4216.208230414 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 100327552 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated63119722 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 trainer/global_step9499 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 100337222 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed171465682681856 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed167392204305954 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8339368219398 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8339282416162 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed163105940979712 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed63119720 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 100337222 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed37407832 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed64619830 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated163105958019072 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 8359829138944 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated167392307148550 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated64620075 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed159052921889792 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8359829138944 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed163490641051648 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated163490665955328 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed63119720 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated171465787158016 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated171465787158016 _runtime4216.208230414 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 100327552 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated63119722 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 trainer/global_step9499 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 100337222 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed171465682681856 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed167392204305954 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8339368219398 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8339282416162 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed163105940979712 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed63119720 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 100337222 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed37407832 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 163456944 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 8359741702144 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 163456944 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed172076323017216 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 8585814729728 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 8585681965568 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 _step7589"2- _wandbruntime4216  _runtime4216@ߛ &loss/total_loss17.854253367482315 epoch0 trainer/global_step9499 0loss/FRt1/assignment_loss2.8535059853033586 0loss/FRt2/assignment_loss2.1019922691312702 % _timestamp1.7857995033865004e+09 _runtime4216.209324025 2loss/SRqqt2/assignment_loss0.8733956770918571 /loss/FRt2/detection_loss2.3219997973435302 .loss/FBt1/detection_loss1.419573730100756 2loss/SRqqt1/assignment_loss0.9866937474968532 ,loss/FBt2/detection_loss2.1143766312993 _step759 3loss/FBt1/assignment_loss0.0001766507323186216 2loss/FBt2/assignment_loss0.004280099951679058 .loss/FRt1/detection_loss2.754911035299301 0loss/SRqqt1/detection_loss1.400765718029881 1loss/SRqqt2/detection_loss1.0225820257022094|=  2loss/SRqqt1/assignment_loss0.9866937474968532 ,loss/FBt2/detection_loss2.1143766312993 _step759 3loss/FBt1/assignment_loss0.0001766507323186216 2loss/FBt2/assignment_loss0.004280099951679058 .loss/FRt1/detection_loss2.754911035299301 0loss/SRqqt1/detection_loss1.400765718029881 1loss/SRqqt2/detection_loss1.0225820257022094 &loss/total_loss17.854253367482315 epoch0 trainer/global_step9499 0loss/FRt1/assignment_loss2.8535059853033586 0loss/FRt2/assignment_loss2.1019922691312702 % _timestamp1.7857995033865004e+09 _runtime4216.209324025 2loss/SRqqt2/assignment_loss0.8733956770918571 /loss/FRt2/detection_loss2.3219997973435302 .loss/FBt1/detection_loss1.419573730100756a(: U gpu.0.gpu41.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts120.367' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent40.12233333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0y[(:  U$ disk.nvme0n1.out182.19775390625 disk.drbd1938.in0) disk./.usagePercent32.34438813614955 disk.sda.out 117.06640625 disk.drbd1673.in0 disk.drbd1673.out0 disk.rbd0.in0 disk.drbd1817.out0 disk.drbd1965.out0 disk.drbd1510.out0 disk.drbd1226.in0 disk.drbd2078.out0 disk.drbd1289.out0 disk.sda1.out0 disk.drbd2078.in0 disk.drbd2118.out0 disk.drbd1356.out0 disk.drbd1828.out0 disk.sda2.out 117.06640625 disk.drbd1817.in0$ disk./.usageGB578.3505096435547! disk.dm-0.out182.19775390625 disk.drbd1356.in0 disk.drbd1931.in0* proc.memory.percent17.182016372680664 network.recv536419 disk.drbd1550.in0 disk.drbd1289.in0 disk.drbd1285.out0 disk.drbd2118.in0 disk.drbd1931.out0 proc.cpu.threads29 disk.drbd1828.in0 disk.drbd1226.out0 disk.drbd2111.in0 disk.rbd1.in0 disk.sda1.in0 disk.rbd1.out 16.8876953125 disk.drbd1965.in0! proc.memory.rssMB 11260.40625 disk.sda.in0 disk.drbd1510.in0 disk.drbd2111.out0* proc.memory.availableMB38507.11328125 disk.drbd1938.out0 disk.rbd0.out0 disk.drbd1285.in0 disk.drbd1584.in0 disk.drbd1584.out0 disk.drbd1552.in0 disk.drbd1552.out0 disk.nvme0n1.in 2.47265625 disk.sda2.in0 disk.drbd2041.in0 disk.drbd1550.out0# memory_percent41.2428081035614 cpu13.783736740962707 network.sent 8.674345e+06 disk.drbd2041.out0 disk.dm-0.in 2.4726562592- _wandbruntime4237  _runtime4237rA trainer/global_step9549 % _timestamp1.7857995246914039e+09 _runtime4237.514344305 _step760 & lr-AdamW/pg10.0002460986087481879 & lr-AdamW/pg20.0002460986087481879:\' _runtime4237.514344305 _step760 & lr-AdamW/pg10.0002460986087481879 & lr-AdamW/pg20.0002460986087481879 trainer/global_step9549 % _timestamp1.7857995246914039e+09LG2- _wandbruntime4237  _runtime4237MuO QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current260 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed63445227 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed163947112367104 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 100845031 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated63445229 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 8402945647616 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed64953969 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated168255586582404 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed37600745 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed27353224 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 8402945647616 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 100854687 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed164333789774848 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated159873207372832 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8382293406448 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 100845031 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 164299916 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed168255483739920 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated163947129406464 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed172349970585088 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 trainer/global_step9549 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed163947112367104 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed63445227 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated172963900248064 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 8402858217984 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated64954231 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_b Cytes.all.freed172963740475904 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 _runtime4237.515756542 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated172350075054080 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated163947129406464 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated63445229 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed159873190333472 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed172349970585088 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 164290258 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated37600747 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated172350075054080 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated164333814678528 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8402858217984 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 100854687 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated27353484 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 8630085569536q'O H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated163947129406464 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated63445229 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed159873190333472 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed172349970585088 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 164290258 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated37600747 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated172350075054080 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated164333814678528 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8402858217984 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 100854687 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated27353484 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 8630085569536 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current260 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed63445227 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed163947112367104 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 100845031 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated63445229 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 8402945647616 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed64953969 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated168255586582404 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed37600745 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed27353224 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 8402945647616 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 100854687 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed164333789774848 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated159873207372832 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8382293406448 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 100845031 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 164299916 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed168255483739920 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated163947129406464 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed172349970585088 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 trainer/global_step9549 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed163947112367104 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed63445227 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated172963900248064 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 8402858217984 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated64954231 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed172963740475904 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 _runtime4237.515756542 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated172350075054080 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 JDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 164317096 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 _runtime4237.881505553 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated27356302 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed27356067 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 100855554 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current235 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 trainer/global_step9549 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated64960986 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed172981854162944 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 _step762 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 8403737837568 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed64960749 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 164317096 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8403825274368 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated172368121443840 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 164307424 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated63451872 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated172368121443840 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated37604684 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8383256682906 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 8630857271296 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed168273101385462 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated159889947545152 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed63451870 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated168273204228058 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed159889930505792 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed164350996891648 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 164307424 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed163964279130112 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated63451872 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 100865224 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated164351021795328 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 8630985841152 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed63451870 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 8403825274368 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current237 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 100865224 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed163964279130112 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 M KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 _step762 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 8403737837568 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed64960749 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 164317096 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8403825274368 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated172368121443840 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 164307424 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated63451872 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated172368121443840 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated37604684 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8383256682906 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 8630857271296 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed168273101385462 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated159889947545152 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed63451870 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated168273204228058 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed159889930505792 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed164350996891648 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 164307424 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed163964279130112 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated63451872 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 100865224 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated164351021795328 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 8630985841152 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed63451870 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 8403825274368 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current237 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 100865224 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed163964279130112 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed37604682 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 164317096 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 _runtime4237.881505553 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated27356302 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed27356067 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 100855554 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current235 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 trainer/global_step9549 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated64960986 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed172981854162944}~2- _wandbruntime4237  _runtime4237  0loss/FRt1/assignment_loss2.8049745712731338 0loss/FRt2/assignment_loss2.2628650693006294 2loss/SRqqt1/assignment_loss1.0615087611232559 4loss/FBt1/assignment_loss1.4375209435884778e-07 3loss/FBt2/assignment_loss2.643594169923932e-06 .loss/FRt2/detection_loss2.518613423025885 0loss/SRqqt1/detection_loss1.701260230395683 trainer/global_step9549 /loss/FBt1/detection_loss0.5541950941962355 % _timestamp1.7857995250597782e+09 1loss/SRqqt2/detection_loss1.2587378288049307 .loss/FBt2/detection_loss1.797412227565313 epoch0 /loss/FRt1/detection_loss2.7555248036577895 _step763 2loss/SRqqt2/assignment_loss0.9330413763084685 %loss/total_loss17.64813617299759 _runtime4237.882607709ɽ .loss/FRt2/detection_loss2.518613423025885 0loss/SRqqt1/detection_loss1.701260230395683 trainer/global_step9549 /loss/FBt1/detection_loss0.5541950941962355 % _timestamp1.7857995250597782e+09 1loss/SRqqt2/detection_loss1.2587378288049307 .loss/FBt2/detection_loss1.797412227565313 epoch0 /loss/FRt1/detection_loss2.7555248036577895 _step763 2loss/SRqqt2/assignment_loss0.9330413763084685 %loss/total_loss17.64813617299759 _runtime4237.882607709 0loss/FRt1/assignment_loss2.8049745712731338 0loss/FRt2/assignment_loss2.2628650693006294 2loss/SRqqt1/assignment_loss1.0615087611232559 4loss/FBt1/assignment_loss1.4375209435884778e-07 3loss/FBt2/assignment_loss2.643594169923932e-06(: U gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts125.517' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent41.839 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors03c^(:  ͺU network.recv537884 disk.sda1.out0 disk.drbd2111.out0 disk.rbd1.in0 disk.drbd1552.out0 disk.drbd1828.out0 disk.drbd1226.out0 disk.drbd1550.out0 disk.drbd2078.out0 disk.drbd1584.in0 disk.sda2.out 117.53515625) proc.memory.percent17.18190312385559 disk.drbd2078.in0 disk.drbd1584.out0 disk.drbd1356.in0 proc.cpu.threads29 network.sent 8.71933e+06 disk.drbd1289.in0 disk.drbd1510.in0! disk.dm-0.out182.39697265625 disk.drbd1817.out0) disk./.usagePercent32.34413959756496 disk.drbd1938.out0 disk.rbd0.out0 disk.drbd2111.in0 disk.drbd1965.in0 disk.drbd1931.in0 disk.drbd1828.in0 disk.drbd1289.out0 cpu13.782895787231938$ disk./.usageGB578.3460655212402 disk.drbd1931.out0 disk.dm-0.in 2.47265625 disk.rbd1.out 16.9814453125 disk.sda.out 117.53515625 disk.nvme0n1.in 2.47265625% memory_percent41.243261098861694 disk.drbd2041.in0 disk.drbd1552.in0 disk.drbd1226.in0 disk.drbd2118.in0 disk.drbd1673.in0 disk.drbd1550.in0 disk.sda.in0 disk.drbd1285.out0 disk.drbd2118.out0 disk.drbd1965.out0 disk.rbd0.in0$ disk.nvme0n1.out182.39697265625 disk.drbd1356.out0$ proc.memory.rssMB11260.33203125 disk.drbd1673.out0* proc.memory.availableMB38506.81640625 disk.drbd1285.in0 disk.drbd1510.out0 disk.sda2.in0 disk.drbd2041.out0 disk.drbd1938.in0 disk.drbd1817.in0 disk.sda1.in0.ȏ2- _wandbruntime4259  _runtime4259$ % _timestamp1.7857995467331097e+09 _runtime4259.556130396 _step764 & lr-AdamW/pg10.0002473872180724532 & lr-AdamW/pg20.0002473872180724532 trainer/global_step9599w & lr-AdamW/pg10.0002473872180724532 & lr-AdamW/pg20.0002473872180724532 trainer/global_step9599 % _timestamp1.7857995467331097e+09 _runtime4259.556130396 _step764ZcP2- _wandbruntime4259  _runtime4259Bw'O L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated169136483661912 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed65293890 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed169136380819428 _runtime4259.557361837 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated65294139 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated173252409339904 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 101373033 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current249 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated37797597 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8426181869956 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 8446854353408 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current247 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated160710215988832 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed27496295 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 165160068 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 101382689 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed164805450517504 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 8446941783040 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed173252304870912 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed173869249649152 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed160710198949472 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated63777379 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated165194170518528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121e'O ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8426181869956 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 8446854353408 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current247 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated160710215988832 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed27496295 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 165160068 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 101382689 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed164805450517504 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 8446941783040 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed173252304870912 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed173869249649152 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed160710198949472 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated63777379 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated165194170518528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated169136483661912 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed65293890 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed169136380819428 _runtime4259.557361837 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated65294139 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated173252409339904 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 101373033 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current249 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated37797597W2- _wandbruntime4259  _runtime42599M EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed65300643 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current241 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed169153998464970 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated173887519259136 _step766 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 8676141623808 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _runtime4259.926476644 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 101393226 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed160726939121792 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 165167576 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 8447733972992 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed173887361591296 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated27499352 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8447821409792 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 165177248 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed165211352731648 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8427059343178 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated160726956161152 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated63784022 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated169154101307566 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated173270455729664 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated164822634319872 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 165177248 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8447733972992 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed63784020 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 101383556 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 8676008859648 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 trainer/global_step9599 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated164822634319872 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712&M A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated63784022 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated169154101307566 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated173270455729664 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated164822634319872 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 165177248 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8447733972992 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed63784020 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 101383556 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 8676008859648 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 trainer/global_step9599 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated164822634319872 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed65300643 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current241 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed169153998464970 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated173887519259136 _step766 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 8676141623808 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _runtime4259.926476644 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 101393226 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed160726939121792 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 165167576 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 8447733972992 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed173887361591296 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated27499352 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8447821409792 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 165177248 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed165211352731648 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8427059343178 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated160726956161152 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856oE42- _wandbruntime4259  _runtime4259Ȇ 1loss/SRqqt1/detection_loss1.5172390568446605 .loss/FBt1/detection_loss1.025241714099358 _step767 /loss/FRt2/assignment_loss1.958566493377453 4loss/FBt1/assignment_loss1.3637174733644471e-05 3loss/FBt2/assignment_loss7.834841765112326e-05 /loss/FBt2/detection_loss1.9186300199168425 % _timestamp1.7857995471048534e+09 /loss/FRt1/assignment_loss2.855641888148749 1loss/SRqqt2/assignment_loss0.962395938619194 .loss/FRt1/detection_loss2.854431921421592 /loss/FRt2/detection_loss2.2328807102834305 0loss/SRqqt2/detection_loss0.956976217740356 epoch0 2loss/SRqqt1/assignment_loss0.8858336060545704 %loss/total_loss17.16792955209859 trainer/global_step9599 _runtime4259.927688494!  4loss/FBt1/assignment_loss1.3637174733644471e-05 3loss/FBt2/assignment_loss7.834841765112326e-05 /loss/FBt2/detection_loss1.9186300199168425 % _timestamp1.7857995471048534e+09 /loss/FRt1/assignment_loss2.855641888148749 1loss/SRqqt2/assignment_loss0.962395938619194 .loss/FRt1/detection_loss2.854431921421592 /loss/FRt2/detection_loss2.2328807102834305 0loss/SRqqt2/detection_loss0.956976217740356 epoch0 2loss/SRqqt1/assignment_loss0.8858336060545704 %loss/total_loss17.16792955209859 trainer/global_step9599 _runtime4259.927688494 1loss/SRqqt1/detection_loss1.5172390568446605 .loss/FBt1/detection_loss1.025241714099358 _step767 /loss/FRt2/assignment_loss1.958566493377453xݐ(: U gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts126.171' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent42.057 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0FZ(:  㺃V disk.drbd1552.out0 disk.sda.out 117.76171875 disk.drbd1356.in0 proc.cpu.threads29 network.recv539477 disk.drbd1550.out0 disk.nvme0n1.in 2.47265625 disk.drbd1965.out0) proc.memory.percent17.18190312385559 network.sent 8.764376e+06 disk.drbd1226.in0 disk.drbd2111.in0 disk.drbd1817.out0 disk.drbd1965.in0$ disk./.usageGB578.3459892272949 disk.drbd2041.out0 disk.drbd1828.in0 disk.drbd1828.out0 disk.drbd1289.out0 disk.rbd0.in0 disk.rbd0.out0 disk.sda.in0) disk./.usagePercent32.34413533080814 disk.drbd2041.in0 disk.drbd1285.in0" disk.nvme0n1.out 183.134765625$ proc.memory.rssMB11260.33203125 disk.rbd1.in0 disk.drbd2078.out0 disk.sda1.out0 disk.dm-0.in 2.47265625 disk.drbd1226.out0 disk.drbd1584.out0 disk.drbd2118.in0 disk.rbd1.out 17.1376953125* proc.memory.availableMB38506.31640625 disk.drbd2118.out0 disk.drbd1356.out0 disk.sda2.out 117.76171875 disk.drbd1817.in0 cpu13.782317595168463 disk.drbd1931.out0 disk.drbd1510.out0 disk.drbd1289.in0 disk.drbd1510.in0 disk.dm-0.out 183.134765625 disk.sda2.in0$ memory_percent41.24402403831482 disk.sda1.in0 disk.drbd1673.in0 disk.drbd1931.in0 disk.drbd1550.in0 disk.drbd1938.in0 disk.drbd2078.in0 disk.drbd1584.in0 disk.drbd1552.in0 disk.drbd1938.out0 disk.drbd2111.out0 disk.drbd1285.out0 disk.drbd1673.out0Tf(: U gpu.0.gpu41.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts95.955' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.984999999999996 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0gY(:  U disk.drbd1552.out0 disk.drbd1285.out0( proc.memory.availableMB 38507.265625 disk.drbd1817.out0 disk.drbd1965.in0 disk.drbd1965.out0 disk.drbd1673.out0 disk.drbd1550.out0 disk.drbd1289.in0 disk.sda1.in0 disk.rbd0.in0* disk./.usagePercent32.344132130740526 disk.drbd1938.out0 disk.drbd1356.out0 disk.drbd1931.in0 disk.drbd1828.in0 disk.drbd1550.in0 disk.drbd1510.out0 disk.drbd2118.out0 disk.sda.out 118.03515625 disk.drbd2078.in0$ memory_percent41.24257564544678$ disk./.usageGB578.3459320068359 disk.drbd1828.out0 disk.sda.in0 disk.drbd1285.in0 disk.sda2.out 118.03515625 disk.drbd1289.out0 disk.sda2.in0 disk.drbd1226.in0 disk.drbd2078.out0" disk.nvme0n1.out 183.330078125 cpu13.781429487454478 disk.drbd1510.in0 disk.dm-0.out 183.330078125 disk.drbd1356.in0 disk.drbd2111.out0 disk.drbd2041.in0 disk.drbd1552.in0 disk.sda1.out0 disk.drbd1584.out0 disk.nvme0n1.in 2.47265625 network.sent 8.809389e+06 disk.dm-0.in 2.47265625 disk.drbd1226.out0 disk.drbd1584.in0 disk.drbd2118.in0) proc.memory.percent17.18190312385559 proc.cpu.threads29 disk.drbd1817.in0 disk.drbd1673.in0 disk.drbd1931.out0 disk.rbd0.out0 network.recv541044 disk.drbd2111.in0 disk.rbd1.in0 disk.drbd1938.in0 disk.rbd1.out 17.1376953125$ proc.memory.rssMB11260.33203125 disk.drbd2041.out0ͱ2- _wandbruntime4281  _runtime4281EB ' lr-AdamW/pg10.00024867582739671854 ' lr-AdamW/pg20.00024867582739671854 trainer/global_step9649 % _timestamp1.7857995686943178e+09 _runtime4281.517275425 _step768\ trainer/global_step9649 % _timestamp1.7857995686943178e+09 _runtime4281.517275425 _step768 ' lr-AdamW/pg10.00024867582739671854 ' lr-AdamW/pg20.00024867582739671854Q(F2- _wandbruntime4281  _runtime4281߶'O YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated174154743625728 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 trainer/global_step9649 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated27639978 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 8720446738944 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated37994447 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated165663805707264 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated170017380741420 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed165663788667904 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 101910691 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated64109529 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed65634168 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 8490937918464 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed174774815422464 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated166054526358528 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated65634425 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed165663788667904 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed64109527 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current257 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated64109529 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed37994445 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated174774973097472 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated174154743625728 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed27639723 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 101901035 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 _step769 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed174154639156736 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 166010562 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 8490850488832 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed166054501454848 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed170017277898936 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8470156136588"O QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 101901035 % _timestamp1.7857995686952631e+09 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated161547224604832 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated165663805707264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current255 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8490850488832 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed161547207565472 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 8490937918464 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed174154639156736 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 166020220 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 8720313967616 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 101910691 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated65634425 GADeviceStatsMonʮ:itor.on_train_batch_start/oversize_segments.current0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed165663788667904 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed64109527 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current257 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated64109529 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed37994445 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated174774973097472 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated174154743625728 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed27639723 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 101901035 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 _step769 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed174154639156736 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 166010562 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 8490850488832 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed166054501454848 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed170017277898936 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8470156136588 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated174154743625728 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 trainer/global_step9649 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated27639978 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 8720446738944 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated37994447 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated165663805707264 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated170017380741420 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed165663788667904 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 101910691 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated64109529 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed65634168 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 8490937918464 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed174774815422464 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated166054526358528 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed161563947737792 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8471033609922 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 166037400 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 166027728 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 8491730108416 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed166071708571648 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed65640965 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 8721224068608 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8470947806686 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated174172790015488 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8491730108416 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 147182080 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8491817545216 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated161563964777152 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 _runtime4281.887258894 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated64116172 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 trainer/global_step9649 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated174172790015488 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated165680972470272 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated64116172 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed64116170 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 101911558 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated165680972470272 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated27642821 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed170034895544478 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 166037400 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current238 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed174172685539328 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated166071733475328 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed27642583 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 8491817545216 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated174793079822336 % _timestamp1.7857995690638423e+09 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated65641205 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8491730108416 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 147182080 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8491817545216 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated161563964777152 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 _runtime4281.887258894 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated64116172 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 trainer/global_step9649 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated174172790015488 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated165680972470272 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated64116172 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed64116170 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 101911558 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated165680972470272 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated27642821 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed170034895544478 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 166037400 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current238 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed174172685539328 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated166071733475328 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed27642583 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 8491817545216 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated174793079822336 % _timestamp1.7857995690638423e+09 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated65641205 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed161563947737792 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8471033609922 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 166037400 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 166027728 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 8491730108416 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed166071708571648 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed65640965 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 8721224068608("2- _wandbruntime4281  _runtime4281!  2loss/SRqqt2/assignment_loss1.0123939745870856 3loss/FBt1/assignment_loss6.296644968619474e-07 1loss/SRqqt2/detection_loss0.9062869939127867 /loss/FBt1/detection_loss0.9754502332936494 /loss/FBt2/detection_loss1.8185374946333468 &loss/total_loss17.693938239825904 _step771 0loss/FRt2/assignment_loss2.2808460223842673 /loss/FRt2/detection_loss2.5578138438958518 $ _timestamp1.785799569065583e+09 _runtime4281.888439316 /loss/FRt1/detection_loss2.4502751917301953 1loss/SRqqt1/detection_loss1.7842285026692681 trainer/global_step9649 0loss/FRt1/assignment_loss2.8566315056203484 2loss/SRqqt1/assignment_loss1.0514316954856946 2loss/FBt2/assignment_loss4.21519489144373e-05 epoch0! 0loss/FRt2/assignment_loss2.2808460223842673 /loss/FRt2/detection_loss2.5578138438958518 $ _timestamp1.785799569065583e+09 _runtime4281.888439316 /loss/FRt1/detection_loss2.4502751917301953 1loss/SRqqt1/detection_loss1.7842285026692681 trainer/global_step9649 0loss/FRt1/assignment_loss2.8566315056203484 2loss/SRqqt1/assignment_loss1.0514316954856946 2loss/FBt2/assignment_loss4.21519489144373e-05 epoch0 2loss/SRqqt2/assignment_loss1.0123939745870856 3loss/FBt1/assignment_loss6.296644968619474e-07 1loss/SRqqt2/detection_loss0.9062869939127867 /loss/FBt1/detection_loss0.9754502332936494 /loss/FBt2/detection_loss1.8185374946333468 &loss/total_loss17.693938239825904 _step771(: ڕU gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts117.329' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent39.10966666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0ZY(:  ̖U disk.drbd1550.in0 network.sent 8.812412e+06 disk.drbd1938.in0 disk.drbd2078.in0 disk.drbd2041.out0 disk.drbd1673.out0 disk.sda1.out0 disk.dm-0.out183.8212890625 disk.drbd1226.in0 disk.drbd1584.out0 proc.cpu.threads29 disk.rbd0.in0 disk.drbd1285.out0 disk.drbd1510.in0) disk./.usagePercent32.34416370474097 disk.drbd1828.in0$ disk./.usageGB578.3464965820312 disk.drbd1817.in0 disk.drbd1550.out0 disk.sda.out 118.2734375 disk.drbd1226.out0 disk.drbd2118.in0 disk.rbd1.in0 disk.nvme0n1.in 2.47265625 disk.drbd1552.in0 disk.sda2.in0 disk.rbd1.out 17.2001953125 disk.drbd1673.in0 disk.dm-0.in 2.47265625 disk.drbd2078.out0 disk.drbd1356.in0 disk.drbd1356.out0) proc.memory.percent17.18190312385559 disk.drbd1931.out0 disk.drbd1510.out0 disk.drbd2111.in0 disk.drbd2111.out0 disk.drbd1552.out0 disk.drbd1285.in0$ proc.memory.rssMB11260.33203125 network.recv542108 disk.drbd1828.out0 disk.drbd1938.out0 disk.drbd1289.out0 disk.drbd1817.out0 disk.drbd2041.in0 disk.rbd0.out0 disk.drbd1584.in0 disk.drbd2118.out0 disk.drbd1289.in0 disk.sda.in0 disk.sda1.in0$ memory_percent41.24224781990051 cpu13.78069106001569 disk.drbd1965.in0 disk.drbd1931.in0# disk.nvme0n1.out183.8212890625* proc.memory.availableMB38507.48046875 disk.sda2.out 118.2734375 disk.drbd1965.out0Ey2- _wandbruntime4303  _runtime4303a % _timestamp1.7857995906310706e+09 _runtime4303.454027685 _step772 ' lr-AdamW/pg10.00024996443672098385 ' lr-AdamW/pg20.00024996443672098385 trainer/global_step9699 ' lr-AdamW/pg10.00024996443672098385 ' lr-AdamW/pg20.00024996443672098385 trainer/global_step9699 % _timestamp1.7857995906310706e+09 _runtime4303.454027685 _step772و.2- _wandbruntime4303  _runtime4303J@3O VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated175057077911552 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed64441677 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 8765679341568 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated65974911 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8534846624256 UGDeviceStatsMonitor.l/%on_train_batch_start/requested_bytes.small_pool.peak 248886712 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed64441677 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 8534934053888 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 trainer/global_step9699 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 166870714 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated166522143857664 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8513958796972 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 166880372 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed27783366 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated166914882198528 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated170898277820928 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 102429037 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed166914857294848 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 % _timestamp1.7857995906319776e+09 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 102438693 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 102438693 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 8765540278784 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8514044600096 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated64441679 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed166522126818304 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated175680561540096 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed162384216181472 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current250 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 102429037 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed175056973442560 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current248 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed65974661 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated27783614 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 _step773 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed175680397573632 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed166522126818304 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed170898174978444 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated64441679 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated38191297^>x'O G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed64441677 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 8534934053888 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 trainer/global_step9699 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 166870714 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated166522143857664 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8513958796972 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 166880372 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed27783366 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated166914882198528 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated170898277820928 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 102429037 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed166914857294848 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 % _timestamp1.7857995906319776e+09 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 102438693 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 102438693 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 8765540278784 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8514044600096 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated64441679 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed166522126818304 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated175680561540096 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed162384216181472 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current250 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 102429037 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed175056973442560 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current248 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed65974661 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated27783614 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 _step773 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed175680397573632 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed166522126818304 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed170898174978444 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated64441679 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated38191297 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated175057077911552 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed64441677 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 8765679341568 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated65974911 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8534846624256 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime4303.819773677 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 102439560 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed166932064411648 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated175075124301312 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated175075124301312 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed175075019825152 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 8535813680640 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 102449230 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 166897552 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated64448322 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed65981440 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 8766446482432 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 166897552 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated175698668561920 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current230 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 8535726243840 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed162400956353792 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 166887880 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8535726243840 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed166539293581312 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated170915895466582 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8514836270194 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated166932089315328 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 % _timestamp1.7857995909963014e+09 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed166539293581312 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 trainer/global_step9699 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 _step774 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated64448322 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 102449230 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 166887880 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated162400973393152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed64448320*M M ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed166539293581312 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated170915895466582 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8514836270194 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated166932089315328 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 % _timestamp1.7857995909963014e+09 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed166539293581312 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 trainer/global_step9699 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 _step774 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated64448322 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 102449230 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 166887880 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 ^JDeviceStatsMonitor.on_train_b91Latch_end/requested_bytes.large_pool.allocated162400973393152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed64448320 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated166539310620672 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed27786208 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8514922073430 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime4303.819773677 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 102439560 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed166932064411648 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated175075124301312 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated175075124301312 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed175075019825152 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 8535813680640 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 102449230 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 166897552 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated64448322 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed65981440 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 8766446482432 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 166897552 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated175698668561920 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current230 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 8535726243840 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed162400956353792 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 166887880 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8535726243840 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0/ 2- _wandbruntime4303  _runtime4303c  3loss/FBt1/assignment_loss1.878447372060884e-07 /loss/FRt2/detection_loss2.1433653044311898 trainer/global_step9699 .loss/FRt1/detection_loss2.638259513924519 1loss/SRqqt1/detection_loss1.9886299038592439 /loss/FBt2/detection_loss1.6999411223456264 /loss/FRt2/assignment_loss2.177345607591712 1loss/SRqqt2/assignment_loss0.840658302951576 3loss/FBt2/assignment_loss0.0008950385361444546 1loss/SRqqt2/detection_loss1.1210756473610122 epoch0 -loss/FRt1/assignment_loss2.5506620321955 .loss/FBt1/detection_loss1.162863281188589 &loss/total_loss17.299888577069126 % _timestamp1.7857995909980447e+09 _runtime4303.820859339 _step775 1loss/SRqqt1/assignment_loss0.976192634839278^ѧ  /loss/FRt2/detection_loss2.1433653044311898 trainer/global_step9699 .loss/FRt1/detection_loss2.638259513924519 1loss/SRqqt1/detection_loss1.9886299038592439 /loss/FBt2/detection_loss1.6999411223456264 /loss/FRt2/assignment_loss2.177345607591712 1loss/SRqqt2/assignment_loss0.840658302951576 3loss/FBt2/assignment_loss0.0008950385361444546 1loss/SRqqt2/detection_loss1.1210756473610122 epoch0 -loss/FRt1/assignment_loss2.5506620321955 .loss/FBt1/detection_loss1.162863281188589 &loss/total_loss17.299888577069126 % _timestamp1.7857995909980447e+09 _runtime4303.820859339 _step775 1loss/SRqqt1/assignment_loss0.976192634839278 3loss/FBt1/assignment_loss1.878447372060884e-07 L(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts118.562' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent39.52066666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0GLtY(:  V disk.drbd1550.out0 disk.drbd2078.out0 disk.drbd1931.in0 disk.drbd1289.in0 disk.drbd1938.out0 disk.drbd2118.out0 disk.drbd2111.out0) proc.memory.percent17.18190908432007 disk.drbd1226.in0 disk.dm-0.in 2.47265625 disk.drbd1584.in0 disk.drbd1285.in0 disk.drbd1673.in0 disk.rbd1.out 17.2001953125 disk.rbd0.in0 disk.sda1.in0 disk.drbd1965.out0 proc.cpu.threads29 disk.drbd1828.in0 disk.drbd1828.out0 disk.sda.in0 disk.drbd1285.out0 disk.drbd1510.in0 disk.dm-0.out183.9501953125 disk.drbd2111.in0 disk.drbd1817.in0$ disk./.usageGB578.3465156555176 disk.drbd1552.out0 disk.rbd0.out0 disk.drbd1510.out0 disk.drbd1584.out0 disk.drbd1673.out0 disk.drbd1552.in0 disk.sda1.out0 disk.drbd2078.in0 disk.sda.out 118.7421875 disk.drbd1817.out0 disk.drbd2041.out0 disk.drbd1356.in0 disk.drbd1356.out0# proc.memory.rssMB 11260.3359375 disk.drbd1289.out0 disk.drbd2118.in0 disk.nvme0n1.in 2.47265625 disk.drbd1965.in0* proc.memory.availableMB38506.91015625 disk.drbd1550.in0 disk.drbd1938.in0# disk.nvme0n1.out183.9501953125 disk.rbd1.in0$ memory_percent41.24311804771423 cpu13.779868195810405 network.sent 8.857643e+06 disk.drbd1931.out0 disk.drbd2041.in0 disk.drbd1226.out0 disk.sda2.in0) disk./.usagePercent32.34416477143018 disk.sda2.out 118.7421875 network.recv543656u w(: ׿U gpu.0.gpu27.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts122.757' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent40.919 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0jW(:  ȉU disk.sda2.in0 disk.drbd2041.in0 disk.sda.in0* proc.memory.availableMB38507.41015625 disk.drbd2111.out0 disk.drbd1828.in0 disk.drbd1285.in0 disk.dm-0.out 184.583984375 disk.drbd1931.in0 disk.drbd1552.out0 disk.drbd1510.in0$ memory_percent41.24235510826111 network.sent 8.902605e+06 disk.drbd1584.out0 disk.drbd1965.out0 disk.drbd1931.out0 disk.drbd1289.in0 disk.rbd0.out0) disk./.usagePercent32.34416263805177$ disk./.usageGB578.3464775085449 disk.drbd1828.out0 disk.sda1.in0 disk.drbd1226.out0 disk.nvme0n1.in 2.47265625 disk.drbd1817.out0 disk.drbd1938.in0 proc.cpu.threads29 disk.drbd2078.in0" disk.nvme0n1.out 184.583984375 cpu13.77922642228389 network.recv544974 disk.drbd1584.in0 disk.sda2.out 119.08203125 disk.drbd2041.out0 disk.drbd1285.out0 disk.drbd1673.out0 disk.drbd1550.out0 disk.sda1.out0 disk.drbd1226.in0 disk.drbd1817.in0 disk.drbd1938.out0) proc.memory.percent17.18190908432007 disk.drbd1289.out0 disk.drbd2078.out0 disk.drbd1356.out0 disk.rbd1.out 17.326171875 disk.drbd1965.in0 disk.drbd1552.in0 disk.drbd1550.in0 disk.drbd2111.in0 disk.rbd0.in0 disk.drbd2118.out0 disk.drbd1356.in0 disk.rbd1.in0 disk.drbd1673.in0 disk.dm-0.in 2.47265625# proc.memory.rssMB 11260.3359375 disk.drbd2118.in0 disk.sda.out 119.08203125 disk.drbd1510.out0't2- _wandbruntime4325  _runtime4325s & lr-AdamW/pg20.0002512530460452491 trainer/global_step9749 % _timestamp1.7857996126840131e+09 _runtime4325.506971112 _step776 & lr-AdamW/pg10.0002512530460452491 % _timestamp1.7857996126840131e+09 _runtime4325.506971112 _step776 & lr-AdamW/pg10.0002512530460452491 & lr-AdamW/pg20.0002512530460452491 trainer/global_step9749 2- _wandbruntime4325  _runtime4325Dhm'O [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 8578930189312 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed171779072057952 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 102966695 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated27927141 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated167775238038528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed27926876 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed175959307728384 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 102966695 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated167380482008064 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed167380464968704 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8578842759680 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 _runtime4325.508251184 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated64773829 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed66315021 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed167775213134848 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 8810833269760 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated64773829 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 102957039 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 8810694206976 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed175959307728384 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 trainer/global_step9749 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 _step777 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed64773827 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 8810694206976 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed175959307728384 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 trainer/global_step9749 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 _step777 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed64773827 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed27926876 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed175959307728384 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 102966695 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated167380482008064 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 TFDeviceStatsMonitor.on_train_batch_start/reserve\(d_bytes.small_pool.peak 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed167380464968704 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8578842759680 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 _runtime4325.508251184 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated64773829 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed66315021xFG2- _wandbruntime4325  _runtime4325;I&M E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed171796689703494 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed167792420251648 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 _step778 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed64780470 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed64780470 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 _runtime4325.873337049 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated167397648771072 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 8579722379264 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed66321821 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8558810536938 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8558724733702 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed167397631731712 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated66322065 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 102967562 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 167757704 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed176604017451008 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated64780472 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated27929981 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current242 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed27929739 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 8811597199360 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated175977458587136 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated64780472 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated38392084 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed38392082 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated167397648771072 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8579722379264 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 167748032 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated171796792546090 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 trainer/global_step9749 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 167748032 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated176604179313152 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 167757704 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 102967562 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated175977458587136 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 102977232 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed175977354110976 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated163237982009152 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0V.&M LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 % _timestamp1.7857996130500238e+09 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 167757704 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed176604017451008 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated64780472 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated27929981 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current242 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed27929739 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 8811597199360 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated175977458587136 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated64780472 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated38392084 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed38392082 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated167397648771072 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8579722379264 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 167748032 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated171796792546090 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 trainer/global_step9749 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 167748032 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated176604179313152 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 167757704 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 102967562 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated175977458587136 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 102977232 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed175977354110976 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated163237982009152 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed171796689703494 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed167792420251648 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 _step778 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed64780470 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed64780470 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 _runtime4325.873337049 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated167397648771072 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 8579722379264 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed66321821 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8558810536938 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8558724733702 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed167397631731712 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated66322065 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 102967562 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 FDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8622838895104 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8601735723988 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 103494697 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated176861746483200 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 _runtime4347.403754287 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 168600676 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed38584995 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 $ _timestamp1.785799634580416e+09 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 _step781 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 103494697 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 103485041 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated66655073 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 8855983498240 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated168238820158464 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed176861642014208 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated168238820158464 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed177491415507456 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed176861642014208 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 103485041kz'O JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated176861746483200 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 _runtime4347.403754287 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 168600676 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed38584995 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 $ _timestamp1.785799634580416e+09 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 _step781 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 103494697 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 103485041 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated66655073 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 8855983498240 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated168238820158464 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed176861642014208 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated168238820158464 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed177491415507456 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed176861642014208 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 103485041 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed66654826 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated176861746483200 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated65105979 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed65105977 trainer/global_step9799 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8622838895104 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8601735723988 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 103494697 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed177509522116608 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated168255986921472 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed176879688396800 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated65112622 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed168652776091648 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 _step782 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated176879792872960 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 trainer/global_step9799 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed66661617 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8623718514688 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 103505234 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated172677689625598 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 168617856 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated168652800995328 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed172677586783002 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 103495564 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated168255986921472 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated164074990625152 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 103505234 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed28072685 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated38588934 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 8856878789120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 8856746024960 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated177509679784448 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated28072943 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 8623718514688 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated65112622 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8602613197210 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current260 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current258 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8602699000446 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 168617856 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed38588932 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 _runtime 4347.78484595 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 8623805951488 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed65112620 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 168608184 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated176879792872960&M >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated168652800995328 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed172677586783002 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 103495564 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated168255986921472 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated164074990625152 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 103505234 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed28072685 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated38588934 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 8856878789120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 8856746024960 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated177509679784448 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated28072943 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 8623718514688 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated65112622 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8602613197210 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current260 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current258 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8602699000446 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 168617856 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed38588932 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 _runtime 4347.78484595 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 8623805951488 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed65112620 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 168608184 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated176879792872960 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 # _timestamp1.78579963496149e+09 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed177509522116608 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated168255986921472 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed176879688396800 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated65112622 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed168652776091648 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 _step782 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated176879792872960 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 trainer/global_step9799 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed66661617 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8623718514688 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 103505234 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated172677689625598 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 168617856r82- _wandbruntime4347  _runtime4347. _step783 .loss/FRt2/detection_loss2.197403513411484 .loss/FBt2/detection_loss2.266143092447344 % _timestamp1.7857996349631202e+09 2loss/SRqqt2/assignment_loss0.8101579506524897 2loss/FBt1/assignment_loss6.93077040100328e-08 2loss/FBt2/assignment_loss8.81968906273802e-05 epoch0 _runtime4347.785942658 /loss/FRt2/assignment_loss2.299231416518145 2loss/SRqqt1/assignment_loss0.9380399496294558 -loss/FRt1/detection_loss2.53323953727196 /loss/FBt1/detection_loss0.3526917302840238 trainer/global_step9799 /loss/FRt1/assignment_loss3.079817216286714 1loss/SRqqt1/detection_loss1.8820198362823142 1loss/SRqqt2/detection_loss1.0461818315162663 %loss/total_loss17.40501434049853 2loss/FBt1/assignment_loss6.93077040100328e-08 2loss/FBt2/assignment_loss8.81968906273802e-05 epoch0 _runtime4347.785942658 /loss/FRt2/assignment_loss2.299231416518145 2loss/SRqqt1/assignment_loss0.9380399496294558 -loss/FRt1/detection_loss2.53323953727196 /loss/FBt1/detection_loss0.3526917302840238 trainer/global_step9799 /loss/FRt1/assignment_loss3.079817216286714 1loss/SRqqt1/detection_loss1.8820198362823142 1loss/SRqqt2/detection_loss1.0461818315162663 %loss/total_loss17.40501434049853 _step783 .loss/FRt2/detection_loss2.197403513411484 .loss/FBt2/detection_loss2.266143092447344 % _timestamp1.7857996349631202e+09 2loss/SRqqt2/assignment_loss0.8101579506524897chu(: U gpu.0.gpu29.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts100.148' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent33.382666666666665 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0CS(:  U disk.drbd2111.in0 disk.drbd2111.out0 disk.drbd1965.in0 disk.drbd1356.out0 network.recv547604 disk.drbd1510.out0 disk.dm-0.in 2.58203125 disk.drbd1289.in0 disk.drbd1510.in0 disk.drbd2118.in0 disk.drbd1931.out0 disk.drbd1550.in0 disk.drbd1285.in0 disk.drbd1673.in0 disk.drbd1828.in0 disk.drbd2078.out0 disk.drbd2118.out0* proc.memory.percent17.182016372680664 disk.drbd1931.in0 disk.drbd1356.in0$ memory_percent41.24301075935364$ disk./.usageGB578.3460578918457 disk.drbd1552.in0 disk.drbd1584.in0 cpu13.777980652328095 disk.sda1.in0 disk.drbd1226.in0) disk./.usagePercent32.34413917088928 disk.sda2.in0 disk.rbd1.in0 disk.drbd2041.in0 disk.drbd2041.out0 disk.drbd1938.in0 disk.drbd1285.out0 disk.drbd1584.out0 disk.drbd1817.in0 disk.sda.in0* proc.memory.availableMB38506.98046875# disk.nvme0n1.out185.0458984375 proc.cpu.threads29 disk.drbd1552.out0 disk.drbd1828.out0 disk.nvme0n1.in 2.58203125 network.sent 8.950587e+06 disk.drbd1817.out0 disk.rbd0.in0 disk.drbd2078.in0 disk.sda2.out 119.71875 disk.rbd0.out0 disk.sda1.out0 disk.dm-0.out185.0458984375 disk.drbd1226.out0 disk.drbd1289.out0 disk.rbd1.out 17.388671875 disk.drbd1965.out0 disk.drbd1673.out0 disk.drbd1550.out0 disk.sda.out 119.71875 disk.drbd1938.out0! proc.memory.rssMB 11260.40625-M(: U gpu.0.gpu24.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts128.315' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.77166666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0*M(:  V disk.drbd1828.out0 disk.drbd1285.in0 disk.drbd1226.out0 disk.rbd1.in0 disk.rbd1.out 17.388671875 disk.drbd1965.in0 disk.drbd1552.in0 disk.sda.in0 proc.cpu.threads29 network.sent 8.995598e+06 disk.drbd1356.out0 disk.drbd1289.in0 disk.drbd1673.out0 disk.drbd1828.in0 disk.drbd1226.in0 disk.rbd0.out0 disk.drbd2118.out0 disk.drbd1965.out0 disk.dm-0.out185.2412109375 disk.drbd2078.out0* disk./.usagePercent32.344140237578486 disk.nvme0n1.in 2.58203125 disk.drbd2041.out0 disk.rbd0.in0 disk.drbd2118.in0 disk.drbd2111.in0 disk.drbd1817.out0 disk.sda.out120.25 disk.drbd1285.out0# disk./.usageGB578.346076965332# disk.nvme0n1.out185.2412109375 disk.drbd1817.in0 disk.drbd1584.out0$ memory_percent41.24301075935364! proc.memory.rssMB 11260.40625 cpu13.777150800370638 disk.drbd1552.out0 disk.drbd1510.in0 disk.drbd1673.in0 disk.drbd1289.out0* proc.memory.availableMB38506.98046875 disk.drbd1356.in0 disk.sda1.in0 disk.drbd1931.out0 disk.drbd2041.in0 disk.drbd1550.out0 disk.drbd1938.in0 disk.sda1.out0 disk.dm-0.in 2.58203125* proc.memory.percent17.182016372680664 network.recv549318 disk.sda2.in0 disk.drbd2111.out0 disk.drbd1550.in0 disk.drbd1938.out0 disk.drbd1584.in0 disk.drbd1510.out0 disk.drbd2078.in0 disk.sda2.out120.25 disk.drbd1931.in0P 2- _wandbruntime4369  _runtime43699l _step784 & lr-AdamW/pg10.0002538302646937797 & lr-AdamW/pg20.0002538302646937797 trainer/global_step9849 % _timestamp1.7857996566317692e+09 _runtime 4369.45477097 " & lr-AdamW/pg10.0002538302646937797 & lr-AdamW/pg20.0002538302646937797 trainer/global_step9849 % _timestamp1.7857996566317692e+09 _runtime 4369.45477097 _step784|x+2- _wandbruntime4369  _runtime4369za'O @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated65438129 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 trainer/global_step9849 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 8901193890304 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated65438129 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 8666922460160 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8645624187496 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed169495924814848 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated177764080769024 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 104022699 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed177763976300032 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated169097158308864 _step785 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 104022699 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated38781847 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated164895259068832 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 104013043 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated177764080769024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed66995589 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8645709990620 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 169451170 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed169097141269504 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 104013043 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed38781845 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated28213997 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed28213744 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed178396985933824 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 J'O E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed66995589 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8645709990620 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 169451170 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed169097141269504 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 104013043 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed38781845 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated28213997 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed28213744 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed178396985933824 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated65438129 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 trainer/global_step9849 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 8901193890304 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated65438129 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 8666922460160 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8645624187496 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed169495924814848 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated177764080769024 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 104022699 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed177763976300032 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated169097158308864 _step785 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 104022699 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated38781847 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated164895259068832 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 104013043 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated177764080769024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130A2- _wandbruntime4369  _runtime4369a'M XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 8667714650112 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 _step786 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated169114325071872 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed67002372 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 169468336 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 104033236 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated164911999241152 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed178415102023168 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed164911982201792 % _timestamp1.7857996570138426e+09 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated38785784 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current256 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated169114325071872 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated178415251302400 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated173558586705106 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated28216844 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed38785782 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 169478008 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated177782127158784 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated65444772 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8646587463954 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 169468336 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 104023566 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8667714650112 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated67002628 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8667802086912 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 104033236 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed177782022682624 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8646501660718 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed65444770 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current254 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 149279232 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 169478008 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated65444772 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 8667802086912 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed173558483862510 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0&M ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated169114325071872 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated178415251302400 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated173558586705106 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated28216844 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed38785782 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 169478008 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated177782127158784 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated65444772 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8646587463954 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 169468336 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 104023566 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8667714650112 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated67002628 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8667802086912 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 104033236 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed177782022682624 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8646501660718 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed65444770 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current254 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 149279232 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 169478008 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated65444772 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 8667802086912 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed173558483862510 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 8667714650112 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 _step786 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated169114325071872 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed67002372 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 169468336 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 104033236 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated164911999241152 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed178415102023168 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed164911982201792 % _timestamp1.7857996570138426e+09 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated38785784 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current256 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130M 2- _wandbruntime4369  _runtime4369tʿ  %loss/total_loss18.74348108075797 2loss/SRqqt1/assignment_loss1.1814030552392496 /loss/FRt1/detection_loss2.7701369173386516 /loss/FRt2/detection_loss2.3160221934318543 % _timestamp1.7857996570154703e+09 _runtime4369.838301943 0loss/FRt1/assignment_loss2.4102478483143974 3loss/FBt2/assignment_loss9.645065630210907e-07 /loss/FBt1/detection_loss1.1435908018969572 epoch0 _step787 0loss/FRt2/assignment_loss2.5376950306050916 4loss/FBt1/assignment_loss1.2348772982634798e-06 0loss/SRqqt1/detection_loss2.370666001649464 1loss/SRqqt2/detection_loss0.8387683413411472 /loss/FBt2/detection_loss2.2888531136241825 trainer/global_step9849 2loss/SRqqt2/assignment_loss0.8860955779331133έ 0loss/SRqqt1/detection_loss2.370666001649464 1loss/SRqqt2/detection_loss0.8387683413411472 /loss/FBt2/detection_loss2.2888531136241825 trainer/global_step9849 2loss/SRqqt2/assignment_loss0.8860955779331133 %loss/total_loss18.74348108075797 2loss/SRqqt1/assignment_loss1.1814030552392496 /loss/FRt1/detection_loss2.7701369173386516 /loss/FRt2/detection_loss2.3160221934318543 % _timestamp1.7857996570154703e+09 _runtime4369.838301943 0loss/FRt1/assignment_loss2.4102478483143974 3loss/FBt2/assignment_loss9.645065630210907e-07 /loss/FBt1/detection_loss1.1435908018969572 epoch0 _step787 0loss/FRt2/assignment_loss2.5376950306050916 4loss/FBt1/assignment_loss1.2348772982634798e-06Bc/R(:  U disk.drbd1226.in0 disk.drbd2078.in0 disk.drbd1584.in0 disk.drbd1965.in0 disk.drbd1289.in0$ memory_percent41.24268889427185 network.recv550230 disk.drbd1226.out0 disk.nvme0n1.in 2.58203125 disk.drbd1510.out0 disk.dm-0.in 2.58203125 cpu13.776526525377845 network.sent 8.998604e+06 disk.drbd1584.out0 disk.drbd1673.out0 disk.drbd1931.in0 disk.drbd1510.in0 disk.drbd1817.in0 disk.rbd1.in0 disk.drbd1552.out0! proc.memory.rssMB 11260.40625 disk.sda2.out120.875 disk.drbd1550.in0 disk.sda1.in0) disk./.usagePercent32.34413917088928 disk.drbd1931.out0 disk.drbd1356.in0 disk.drbd1828.out0 disk.drbd1938.in0 disk.drbd1938.out0 disk.drbd2041.out0 disk.drbd1828.in0 disk.drbd1550.out0! disk.dm-0.out185.56103515625 disk.drbd2078.out0 disk.sda2.in0 disk.drbd1965.out0 disk.sda1.out0 disk.drbd1285.in0$ disk.nvme0n1.out185.56103515625 disk.drbd1552.in0 disk.drbd1289.out0 disk.drbd2111.in0 proc.cpu.threads29 disk.drbd2118.in0 disk.drbd2118.out0 disk.drbd2111.out0 disk.rbd1.out 17.5146484375 disk.drbd1673.in0 disk.rbd0.in0 disk.drbd1356.out0 disk.drbd1817.out0 disk.drbd2041.in0 disk.rbd0.out0 disk.sda.out120.875 disk.drbd1285.out0 disk.sda.in0* proc.memory.availableMB38507.19140625* proc.memory.percent17.182016372680664$ disk./.usageGB578.34605789184573(: ݯU gpu.0.gpu33.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts127.779' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent42.592999999999996 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0+2- _wandbruntime4391  _runtime4391 f & lr-AdamW/pg20.0002551188740180451 trainer/global_step9899 % _timestamp1.7857996787583597e+09 _runtime4391.581326544 _step788 & lr-AdamW/pg10.0002551188740180451Z@) & lr-AdamW/pg10.0002551188740180451 & lr-AdamW/pg20.0002551188740180451 trainer/global_step9899 % _timestamp1.7857996787583597e+09 _runtime4391.581326544 _step7882- _wandbruntime4391  _runtime4391*k'O MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed67336305 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed38978695 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 8946294569984 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 % _timestamp1.7857996787592816e+09 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed179302575224832 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 _step789 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 104541045 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current249 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed178666310585856 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed178666310585856 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated65770279 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated174421866138960 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 trainer/global_step9899 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 170311322 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 104550701 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 104550701 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated65770279 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed169955479419904 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 8710831165952 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8689598454128 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated169955496459264 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated178666415054848 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed170356280654848 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated165732267684832 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 170320980 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8710831165952 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated169955496459264 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed28357610 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated170356305558528 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 104541045 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated28357859 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 8946427341312 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed165732250645472 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8689512651004 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed174421763296476 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032Q}O VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 104541045 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated28357859 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 8946427341312 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed165732250645472 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8689512651004 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed174421763296476 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed67336305 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed38978695 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 8946294569984 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 % _timestamp1.7857996787592816e+09 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed179302575224832 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 _step789 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 104541045 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current249 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed178666310585856 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed178666310585856 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated65770279 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated174421866138960 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 trainer/global_step9899 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 170311322 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 104550701 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 104550701 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated65770279 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed169955479419904 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 8710831165952 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8689598454128 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated169955496459264 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated178666415054848 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed170356280654848 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated165732267684832 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 170320980 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8710831165952 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated169955496459264 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed28357610 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated170356305558528 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121OI2- _wandbruntime4391  _runtime4391 #&M OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 104551568 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed165748990817792 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed65776920 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 8947196284416 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated67343309 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated174439483784614 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 # _timestamp1.78579967912368e+09 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated65776922 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 trainer/global_step9899 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated170373512675328 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed178684356968448 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 170338160 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 170328488 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8711710785536 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 170328488 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8690475927462 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 170338160 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 104551568 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed65776920 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated28360675 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed169972646182912 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed38982632 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8711798222336 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed179320684056064 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 8711710785536 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 104561238 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated178684461444608 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed67343063 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 _step790 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8690390124226 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated65776922 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 trainer/global_step9899 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated170373512675328 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed178684356968448 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 170338160 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 170328488 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8711710785536 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 170328488 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8690475927462 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 170338160 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 104551568 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed65776920 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated28360675 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed169972646182912 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed38982632 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8711798222336 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed179320684056064 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 8711710785536 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 104561238 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated178684461444608 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed67343063 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 _step790 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8690390124226 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 104551568 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed165748990817792Ta2- _wandbruntime4391  _runtime4391f % _timestamp1.7857996791252964e+09 0loss/FRt1/assignment_loss2.5276869581295895 2loss/FBt2/assignment_loss0.011650028280197025 _runtime4391.948127724 /loss/FRt2/assignment_loss2.329665848086862 1loss/SRqqt1/assignment_loss0.973453978923599 3loss/FBt1/assignment_loss0.0002953069179194713 /loss/FRt2/detection_loss2.3909203155952343 1loss/SRqqt2/detection_loss1.0188659546676426 /loss/FBt1/detection_loss0.6658307045811054 trainer/global_step9899 2loss/SRqqt2/assignment_loss0.9838938206116107 1loss/SRqqt1/detection_loss1.4037243389155203 /loss/FBt2/detection_loss2.2022877133396617 epoch0 _step791 /loss/FRt1/detection_loss2.5463900392254195 %loss/total_loss17.05466500727436B5  /loss/FRt2/assignment_loss2.329665848086862 1loss/SRqqt1/assignment_loss0.973453978923599 3loss/FBt1/assignment_loss0.0002953069179194713 /loss/FRt2/detection_loss2.3909203155952343 1loss/SRqqt2/detection_loss1.0188659546676426 /loss/FBt1/detection_loss0.6658307045811054 trainer/global_step9899 2loss/SRqqt2/assignment_loss0.9838938206116107 1loss/SRqqt1/detection_loss1.4037243389155203 /loss/FBt2/detection_loss2.2022877133396617 epoch0 _step791 /loss/FRt1/detection_loss2.5463900392254195 %loss/total_loss17.05466500727436 % _timestamp1.7857996791252964e+09 0loss/FRt1/assignment_loss2.5276869581295895 2loss/FBt2/assignment_loss0.011650028280197025 _runtime4391.948127724(: U gpu.0.gpu0.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts126.393' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent42.131 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0cmT(:  U disk.drbd1356.out0 disk.drbd1938.out0 disk.rbd0.in0 disk.drbd1965.out0 disk.sda.out121.3125 disk.drbd1550.in0 disk.drbd1510.in0 disk.drbd1226.in0 disk.drbd2111.in0 disk.drbd1965.in0* proc.memory.availableMB38506.91015625 disk.drbd1584.out0 disk.drbd1552.out0$ disk./.usageGB578.3460006713867 disk.drbd2118.out0 disk.drbd2111.out0 disk.rbd1.out 17.5146484375 disk.drbd2041.out0 disk.drbd1828.in0 disk.drbd1510.out0$ memory_percent41.24311804771423 disk.drbd2078.out0 disk.nvme0n1.in 2.58203125 disk.drbd1931.in0 disk.sda1.out0 disk.sda2.in0 disk.drbd1817.out0 disk.drbd1673.in0 disk.drbd1552.in0 disk.rbd0.out0 disk.sda.in0 disk.sda1.in0 disk.rbd1.in0 disk.drbd1673.out0 network.recv551870 disk.drbd1356.in0 disk.drbd1289.in0! disk.dm-0.out185.70166015625! proc.memory.rssMB 11260.40625 disk.sda2.out121.3125* proc.memory.percent17.182016372680664 disk.drbd1584.in0 disk.drbd2118.in0 disk.drbd1938.in0 disk.drbd1285.in0 disk.drbd1285.out0 network.sent 9.043627e+06) disk./.usagePercent32.34413597082166$ disk.nvme0n1.out185.70166015625 disk.drbd1931.out0 disk.drbd1828.out0 disk.dm-0.in 2.58203125 cpu13.776915616059307 proc.cpu.threads29 disk.drbd1817.in0 disk.drbd1550.out0 disk.drbd1289.out0 disk.drbd2041.in0 disk.drbd1226.out0 disk.drbd2078.in0k(: χU gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts95.886' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.961999999999996 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0ׄ[(:  U disk.drbd1938.in0 disk.drbd2118.in0 disk.drbd1673.in0 disk.drbd1552.out0 disk.drbd1550.in0 disk.sda2.out 121.76953125) disk./.usagePercent32.34413917088928$ disk./.usageGB578.3460578918457 disk.sda1.out0 disk.drbd1510.in0! disk.dm-0.out186.00146484375 disk.sda2.in0* proc.memory.percent17.182016372680664 disk.drbd1931.in0 disk.drbd2078.in0 disk.drbd1817.out0 disk.sda1.in0 disk.drbd1356.out0$ memory_percent41.24311804771423 disk.drbd1673.out0 disk.drbd1510.out0$ disk.nvme0n1.out186.00146484375 disk.drbd1289.out0 disk.drbd1584.out0 disk.drbd1965.in0 disk.drbd1931.out0 disk.drbd1828.in0 disk.drbd1938.out0 disk.dm-0.in 2.58203125 disk.drbd2111.out0 disk.rbd0.in0 disk.sda.in0 disk.drbd1226.in0 disk.drbd2078.out0 disk.rbd1.out 17.5771484375 proc.cpu.threads29 disk.drbd1552.in0 disk.drbd2118.out0 disk.drbd1817.in0 disk.rbd1.in0 disk.drbd1289.in0 disk.rbd0.out0 disk.drbd1965.out0 disk.sda.out 121.76953125 disk.drbd1285.in0! proc.memory.rssMB 11260.40625 cpu13.77771874328304 network.sent 9.088659e+06 network.recv553556 disk.drbd2041.in0 disk.drbd2041.out0 disk.drbd1356.in0 disk.drbd2111.in0 disk.drbd1285.out0 disk.drbd1584.in0 disk.nvme0n1.in 2.58203125 disk.drbd1828.out0 disk.drbd1550.out0 disk.drbd1226.out0* proc.memory.availableMB38506.91015625ǫ2- _wandbruntime4414  _runtime4414-;  % _timestamp1.7857997013334527e+09 _runtime4414.156435597 _step792 ' lr-AdamW/pg10.00025640748334231035 ' lr-AdamW/pg20.00025640748334231035 trainer/global_step9949B ' lr-AdamW/pg10.00025640748334231035 ' lr-AdamW/pg20.00025640748334231035 trainer/global_step9949 % _timestamp1.7857997013334527e+09 _runtime4414.156435597 _step7924M2- _wandbruntime4414  _runtime4414ѱF$O TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed28501861 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated66102429 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed166569259261472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JBD$eviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 % _timestamp1.7857997013343832e+09 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated28502122 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed66102427 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed179568644871680 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 171181132 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed39175545 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed180208159156224 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed175302660375984 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated170813834609664 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 105069047 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated179568749340672 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed170813817570304 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated166569276300832 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 105069047 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed179568644871680 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated66102429 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 171171474 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8733401114512 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed66102427 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 8991657529856 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 _step793 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated170813834609664 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated39175547 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8754827301376 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current263 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated175302763218468 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 105078703 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 8754827301376 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed171216636494848 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed170813817570304 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 8754914731008 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current261 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated171216661398528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 105078703 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated179568749340672 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated67677669 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed67677406 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated180208318928384 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8733486917636 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 _runtime4414.157674837 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 8754914731008M 'O YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 _runtime4414.157674837 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 8754914731008 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed28501861 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated66102429 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed166569259261472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 % _timestamp1.7857997013343832e+09 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated28502122 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed66102427 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed179568644871680 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 171181132 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed39175545 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed180208159156224 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed175302660375984 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated170813834609664 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 105069047 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated179568749340672 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed170813817570304 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated166569276300832 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 105069047 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed179568644871680 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated66102429 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 171171474 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8733401114512 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed66102427 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 8991657529856 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 _step793 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated170813834609664 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated39175547 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8754827301376 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current263 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated175302763218468 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 105078703 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 8754827301376 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed171216636494848 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed170813817570304 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 8754914731008 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current261 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated171216661398528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 105078703 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated179568749340672 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated67677669 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed67677406 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated180208318928384 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 87334869176362- _wandbruntime4414  _runtime4414D=&M A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed66109070 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 8992571916288 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed170830984333312 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed170830984333312 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated175320380864122 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed175320278021526 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated179586795730432 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated66109072 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8734278587734 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 trainer/global_step9949 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 171198312 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 _runtime4414.528259705 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 $ _timestamp1.785799701704917e+09 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 105089240 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed67684261 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 105079570 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8755794357760 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated170831001372672 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 8755794357760 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated28505027 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 171188640 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed39179482 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated171233868515328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 105079570 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed171233843611648 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 105089240 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8734364390970 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 171188640 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated166586016473152 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _step794 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 171198312 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated67684511 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current250 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed179586691254272 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated66109072 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed166585999433792 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current170393607T? M E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated171233868515328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 105079570 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed171233843611648 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 105089240 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8734364390970 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 171188640 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated166586016473152 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _step794 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 171198312 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated67684511 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current250 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 S?DeviceStatsMonitor.onU ?_train_batch_end/allocated_bytes.all.freed179586691254272 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated66109072 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed166585999433792 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed66109070 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 8992571916288 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed170830984333312 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed170830984333312 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated175320380864122 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed175320278021526 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated179586795730432 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated66109072 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8734278587734 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 trainer/global_step9949 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 171198312 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 _runtime4414.528259705 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 $ _timestamp1.785799701704917e+09 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 105089240 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed67684261 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 105079570 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8755794357760 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated170831001372672 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 8755794357760 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated28505027 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 171188640 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed39179482 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 302199603222- _wandbruntime4414  _runtime4414~w /loss/FBt1/detection_loss0.9877127393286813 3loss/FBt1/assignment_loss3.752868469977902e-07 2loss/FBt2/assignment_loss0.001478074253067906 1loss/SRqqt1/detection_loss1.8601497308435766 1loss/SRqqt2/detection_loss0.8929423735738774 0loss/FRt1/assignment_loss3.0274651217751387 0loss/FRt2/assignment_loss2.6461476619179183 .loss/FBt2/detection_loss2.379175461955707 epoch0 trainer/global_step9949 %loss/total_loss18.70426017870937 2loss/SRqqt1/assignment_loss1.1244628380941735 2loss/SRqqt2/assignment_loss0.8249606923614872 /loss/FRt1/detection_loss2.4764609522208936 +loss/FRt2/detection_loss2.483304157098 $ _timestamp1.785799701706666e+09 _runtime4414.529506176 _step795m0f  /loss/FBt1/detection_loss0.9877127393286813 3loss/FBt1/assignment_loss3.752868469977902e-07 2loss/FBt2/assignment_loss0.001478074253067906 1loss/SRqqt1/detection_loss1.8601497308435766 1loss/SRqqt2/detection_loss0.8929423735738774 0loss/FRt1/assignment_loss3.0274651217751387 0loss/FRt2/assignment_loss2.6461476619179183 .loss/FBt2/detection_loss2.379175461955707 epoch0 trainer/global_step9949 %loss/total_loss18.70426017870937 2loss/SRqqt1/assignment_loss1.1244628380941735 2loss/SRqqt2/assignment_loss0.8249606923614872 /loss/FRt1/detection_loss2.4764609522208936 +loss/FRt2/detection_loss2.483304157098 $ _timestamp1.785799701706666e+09 _runtime4414.529506176 _step795Fֲܜ(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts107.401' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent35.800333333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0PT1[(:  žU disk.drbd1226.out0 disk.drbd1673.out0 disk.sda1.in0 disk.nvme0n1.in 2.58203125 disk.drbd1550.in0 disk.drbd1938.out0 disk.sda2.out 122.1484375 disk.rbd1.in0 disk.drbd2041.out0 disk.dm-0.in 2.58203125 disk.drbd2041.in0* proc.memory.availableMB38506.84765625 disk.sda1.out0 disk.drbd1584.in0 disk.drbd1817.in0 disk.drbd1931.in0 disk.drbd1550.out0* proc.memory.percent17.182016372680664 network.recv554589% memory_percent41.243213415145874* disk./.usagePercent32.344140237578486 disk.drbd1510.in0 disk.drbd1510.out0! disk.dm-0.out186.18505859375 disk.drbd2078.out0 disk.drbd1552.in0 disk.drbd1828.in0# disk./.usageGB578.346076965332 disk.drbd1356.out0 disk.drbd1673.in0 disk.drbd1226.in0 disk.drbd2078.in0$ disk.nvme0n1.out186.18505859375 disk.drbd1965.out0 disk.drbd1931.out0 disk.drbd1938.in0 disk.rbd0.in0 cpu13.778849067022154 disk.drbd1285.out0 disk.drbd1584.out0 disk.drbd1289.out0 disk.sda.out 122.1484375 disk.drbd2118.out0 disk.drbd2111.in0 disk.drbd2118.in0 disk.rbd0.out0 disk.drbd1965.in0 disk.drbd1828.out0 disk.drbd1289.in0 network.sent 9.091684e+06 disk.sda2.in0 disk.rbd1.out 17.5771484375 disk.drbd1285.in0 disk.drbd1356.in0 disk.drbd2111.out0 disk.drbd1817.out0 disk.drbd1552.out0 disk.sda.in0! proc.memory.rssMB 11260.40625 proc.cpu.threads29߲xT2- _wandbruntime4436  _runtime4436 ؐ ' lr-AdamW/pg10.00025769609266657566 ' lr-AdamW/pg20.00025769609266657566 trainer/global_step9999 $ _timestamp1.785799723331337e+09 _runtime4436.154331326 _step796M _runtime4436.154331326 _step796 ' lr-AdamW/pg10.00025769609266657566 ' lr-AdamW/pg20.00025769609266657566 trainer/global_step9999 $ _timestamp1.785799723331337e+09C+Ӌ2- _wandbruntime4436  _runtime4436r|'O M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed68016985 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current258 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated172077017238528 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 trainer/global_step9999 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 105597049 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated28644848 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed66434577 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed181113718058496 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed28644590 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 172041284 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated180471083626496 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed171672155720704 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed172076992334848 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current260 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8777375381144 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 8798910866432 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 % _timestamp1.7857997233323207e+09 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated66434579 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 105606705 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8777289578020 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 172031626 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed180470979157504 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated171672172760064 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 8798910866432 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 % _timestamp1.7857997233323207e+09 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated66434579 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 105606705 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8777289578020 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 172031626 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed180470979157504 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated171672172760064 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed66434577 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816WM2- _wandbruntime4436  _runtime4436cM ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed171689322483712 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8778252854478 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 9037761420800 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 _runtime4436.519877669 _st!ep798 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated68024021 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated181131985776128 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed167423008049792 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 105617242 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8799790493184 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 9037630753792 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 trainer/global_step9999 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 172058464 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 105617242 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed39376332 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 105607572 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed172094199451648 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 172048792 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed66441220 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 $ _timestamp1.785799723696547e+09 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current252 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated171689339523072 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8799703056384 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated180489130016256 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated66441222 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated39376334 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated180489130016256 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed68023769 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated172094224355328 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed171689322483712 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 8799703056384 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated28647687 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032hh&M \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 8799790493184 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current250 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed181131830205440 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed171689322483712 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 8799703056384 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated28647687 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed171689322483712 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8778252854478 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 9037761420800 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 _runtime4436.519877669 _step798 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated68024021 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated181131985776128 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed167423008049792 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 105617242 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8799790493184 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 9037630753792 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 trainer/global_step9999 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 172058464 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 105617242 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed39376332 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 105607572 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed172094199451648 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 172048792 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed66441220 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 $ _timestamp1.785799723696547e+09 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current252 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated171689339523072 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8799703056384 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated180489130016256 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated66441222 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated39376334 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated180489130016256 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed68023769 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated172094224355328 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 FDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed66766727 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed28788534 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 106134707 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated28788794 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated172937373078528 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 106125051 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated177064557377484 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated168243293532832 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed168243276493472 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 8842907001856 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated66766729 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed68357779 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8821263844652 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed181373313443328 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 % _timestamp1.7857997457584674e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed66766727 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated172530510910464 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed172937348174848 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed66766727 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed28788534 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 106134707 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated28788794 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated172937373078528 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 106125051 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated177064557377484 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated168243293532832 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed168243276493472 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 8842907001856 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated66766729 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed68357779 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8821263844652 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed181373313443328 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 % _timestamp1.7857997457584674e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed66766727 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated172530510910464 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed172937348174848 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 2- _wandbruntime4458  _runtime4458lH&M E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed172547660634112 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated172954580195328 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated39573184 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed182037377553920 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 172918616 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed66773370 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed68364563 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated172547677673472 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 8843699191808 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated66773372 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 9082950832128 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8843699191808 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8843786628608 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed172547660634112 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 172908944 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated181391464302080 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed172954555291648 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated177082175023138 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed168260016665792 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 106135574 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current249 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed39573182 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated182037531027456 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed28791381 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 9082822262272 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated172547677673472 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed66773370 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated168260033705152 trainer/global_step10049 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed172547660634112 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated172954580195328 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated39573184 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed182037377553920 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 172918616 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed66773370 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed68364563 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated172547677673472 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 8843699191808 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated66773372 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 9082950832128 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8843699191808 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8843786628608 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed172547660634112 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 172908944 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated181391464302080 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed172954555291648 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated177082175023138 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed168260016665792 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 106135574 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current249 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed39573182 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated182037531027456 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed28791381 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 9082822262272 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated172547677673472 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed66773370 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated168260033705152 trainer/global_step10049 FDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8865152308160 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated182275752198144 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 8886903137280 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 106662709 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed173797704014848 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed182275647729152 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 106662709 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 173761588 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed177945351614508 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated67098879 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed68697432 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current234 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 _step805 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 106653053 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed182924802594816 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed67098877 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated182924953978368 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8886815707648 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 8886903137280 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated182275752198144 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated173388849060864 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated169080302148832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680"a'O G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8886815707648 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 8886903137280 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated182275752198144 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated173388849060864 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated169080302148832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed39766095 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated173797728918528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 106653053 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 126479872 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated177945454456992 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current236 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated68697668 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated173388849060864 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed173388832021504 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed169080285109472 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8865066505036 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 173751930 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated39766097 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 9127098579968 _runtime4480.732833306 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9127225059840 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 trainer/global_step10099 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed173388832021504 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 8886815707648 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated67098879 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated28931571 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed28931337 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8865152308160 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated182275752198144 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 8886903137280 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 106662709 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed173797704014848 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed182275647729152 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 106662709 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 173761588 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed177945351614508 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated67098879 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed68697432 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current234 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 _step805 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 106653053 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed182924802594816 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed67098877 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated182924953978368 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0#2- _wandbruntime4481  _runtime4481A0t M C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed67105520 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated67105522 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 106663576 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed173814911131648 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated169097042321152 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 173778768 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 106673246 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated182293798587904 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated67105522 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 9127997203968 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 173769096 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8887782764032 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 trainer/global_step10099 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 106663576 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _step806 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed169097025281792 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated68704475 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8866029781494 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed173405998784512 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 9128134162432 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated173406015823872 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed68704206 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated177963072102646 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 % _timestamp1.7857997682871878e+09 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 173778768 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated39770034 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 _runtime4481.110534779 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed182942908335616 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251y=&M D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 173769096 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8887782764032 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 trainer/global_step10099 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 106663576 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _step806 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed169097025281792 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated68704475 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8866029781494 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed173405998784512 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 9128134162432 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated173406015823872 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed68704206 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated177963072102646 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 % _timestamp1.7857997682871878e+09 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 173778768 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated39770034 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 _runtime4481.110534779 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed182942908335616 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed67105520 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated67105522 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 106663576 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed173814911131648 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated169097042321152 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 173778768 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 106673246 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated182293798587904 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated67105522 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 9127997203968 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 2488867126,/2- _wandbruntime4481  _runtime4481L  4loss/FBt2/assignment_loss1.8405182261934292e-05 1loss/SRqqt2/detection_loss1.0365242295632404 .loss/FBt2/detection_loss2.648895686864853 trainer/global_step10099 _runtime4481.111662808 /loss/FRt1/detection_loss2.6116474235986735 1loss/SRqqt1/detection_loss1.9882968557523746 &loss/total_loss18.831999003579135 3loss/FBt1/assignment_loss0.0009793273570699744 /loss/FRt2/detection_loss2.1885410123146496 /loss/FBt1/detection_loss1.3421795828556964 /loss/FRt1/assignment_loss2.693143270232461 .loss/FRt2/assignment_loss2.36093709560541 2loss/SRqqt1/assignment_loss1.0291990282050534 epoch0 % _timestamp1.7857997682888315e+09 _step807 2loss/SRqqt2/assignment_loss0.9316370860473918f_ 1loss/SRqqt1/detection_loss1.9882968557523746 &loss/total_loss18.831999003579135 3loss/FBt1/assignment_loss0.0009793273570699744 /loss/FRt2/detection_loss2.1885410123146496 /loss/FBt1/detection_loss1.3421795828556964 /loss/FRt1/assignment_loss2.693143270232461 .loss/FRt2/assignment_loss2.36093709560541 2loss/SRqqt1/assignment_loss1.0291990282050534 epoch0 % _timestamp1.7857997682888315e+09 _step807 2loss/SRqqt2/assignment_loss0.9316370860473918 4loss/FBt2/assignment_loss1.8405182261934292e-05 1loss/SRqqt2/detection_loss1.0365242295632404 .loss/FBt2/detection_loss2.648895686864853 trainer/global_step10099 _runtime4481.111662808 /loss/FRt1/detection_loss2.6116474235986735 n(: ЫU gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts131.995' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent43.998333333333335 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0ЇU(:  U disk.drbd1510.out0 disk.drbd1965.in0 disk.drbd1550.out0 disk.drbd1285.out0 disk.drbd2078.out0 cpu13.782752937135943 disk.nvme0n1.in 4.64453125 disk.drbd1356.out0 disk.sda2.in0! proc.memory.rssMB 11260.40625 disk.drbd1289.in0 network.sent 9.230028e+06 disk.drbd1550.in0 disk.drbd1285.in0 disk.drbd1226.in0 disk.drbd2111.out0 disk.drbd1828.out0 disk.drbd1673.in0 disk.drbd1289.out0 disk.sda.in0! disk.dm-0.out187.33740234375 disk.sda2.out 123.6015625 network.recv560367) disk./.usagePercent32.34416839817347 disk.rbd0.out0 disk.dm-0.in 4.64453125 disk.drbd1226.out0 disk.rbd0.in0 disk.sda.out 123.6015625 disk.rbd1.out 17.765625 disk.drbd1828.in0 disk.drbd1673.out0 disk.rbd1.in0( proc.memory.availableMB 38506.390625 disk.drbd2041.out0 disk.drbd1938.in0 disk.drbd1510.in0 disk.drbd2118.in0 disk.drbd1817.out0 disk.sda1.in0 disk.sda1.out0$ disk./.usageGB578.3465805053711 disk.drbd1931.out0 disk.drbd1938.out0 disk.drbd2111.in0 disk.drbd1552.in0* proc.memory.percent17.182016372680664 disk.drbd1931.in0 disk.drbd2041.in0% memory_percent41.243910789489746 disk.drbd2078.in0 disk.drbd1584.in0 disk.drbd1584.out0 disk.drbd2118.out0 disk.drbd1817.in0 proc.cpu.threads29 disk.drbd1552.out0$ disk.nvme0n1.out187.33740234375 disk.drbd1356.in0 disk.drbd1965.out0A(: 졹U gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts107.203' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent35.73433333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0L(:  ŠV network.sent 9.275018e+06 disk.drbd1356.in0 disk.sda2.in0 disk.drbd1673.in0 disk.drbd1550.in0 disk.sda.in0 disk.sda1.in0$ proc.memory.availableMB38505.75$ memory_percent41.24488830566406 disk.dm-0.out 188.923828125 disk.sda2.out 124.0390625 disk.drbd1938.in0 disk.drbd1938.out0 disk.sda.out 124.0390625 disk.drbd2118.in0 disk.nvme0n1.in 4.64453125 disk.drbd2111.in0 disk.drbd1965.out0 cpu13.78417624049144$ disk./.usageGB578.3465614318848 disk.rbd1.out 18.0078125 disk.drbd1552.out0 disk.dm-0.in 4.64453125 disk.drbd1226.out0 disk.drbd2078.out0 disk.drbd1552.in0 disk.drbd1226.in0 disk.drbd1828.in0 proc.cpu.threads29 disk.drbd1550.out0 disk.sda1.out0" disk.nvme0n1.out 188.923828125 disk.rbd1.in0 disk.drbd1931.in0 disk.drbd2041.out0 disk.drbd1289.out0! proc.memory.rssMB 11260.40625 network.recv561941 disk.drbd1817.in0 disk.drbd1931.out0 disk.drbd1285.out0 disk.drbd1584.out0 disk.rbd0.in0* proc.memory.percent17.182016372680664 disk.drbd1289.in0 disk.drbd1584.in0 disk.drbd2118.out0) disk./.usagePercent32.34416733148427 disk.drbd1285.in0 disk.drbd1510.in0 disk.drbd1510.out0 disk.drbd2111.out0 disk.drbd2041.in0 disk.drbd2078.in0 disk.drbd1356.out0 disk.drbd1817.out0 disk.drbd1965.in0 disk.drbd1673.out0 disk.drbd1828.out0 disk.rbd0.out0e3 2- _wandbruntime4502  _runtime45021 % _timestamp1.7857997900963461e+09 _runtime4502.919354787 _step808 & lr-AdamW/pg10.0002615619206393716 & lr-AdamW/pg20.0002615619206393716 trainer/global_step10149zb & lr-AdamW/pg20.0002615619206393716 trainer/global_step10149 % _timestamp1.7857997900963461e+09 _runtime4502.919354787 _step808 & lr-AdamW/pg10.0002615619206393716$2- _wandbruntime4502  _runtime4502h'O MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed169917293725472 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9172424138752 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed29074737 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated183178086483968 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 9172287173120 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated183178086483968 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current265 trainer/global_step10149 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated29075000 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed183177982014976 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 8930899272704 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 107181055 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed174658059854848 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 _runtime4502.920552693 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 8930811843072 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed69037682 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 % _timestamp1.7857997900973256e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 8930899272704 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed183830347027968 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated174247187211264 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated39962947 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8930811843072 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated67431029 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current263 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated174658084758528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed39962945 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed183177982014976 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated174247187211264 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated178826351536500~QO JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 107181055 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed174658059854848 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 _runtime4502.920552693 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 8930811843072 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed69037682 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 % _timestamp1.7857997900973256e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 8930899272704 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed183830347027968 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated174247187211264 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated39962947 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8930811843072 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated67431029 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current263 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated174658084758528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed39962945 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed183177982014976 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated174247187211264 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated178826351536500 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed169917293725472 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9172424138752 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.smald׋l_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed29074737 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated183178086483968 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 9172287173120 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated183178086483968 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current265 trainer/global_step10149 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated29075000 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed183177982014976 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 8930899272704 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658]Xa2- _wandbruntime4503  _runtime4503U&M D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 174629248 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated67437672 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8931778899456 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed174675266971648 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8931691462656 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated67437672 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 174638920 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 107201248 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated174675291875328 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _runtime4503.289386684 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 8931778899456 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated29077875 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8909918245002 trainer/global_step10149 $ _timestamp1.785799790466038e+09 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated183196132873728 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed69044501 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated39966884 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed174264336934912 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 107191578 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed39966882 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 107191578 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed67437670 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated174264353974272 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated183848614942208 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed67437670 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8909832441766 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0nP&M L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed67437670 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated174264353974272 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated183848614942208 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed67437670 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 8909832441766 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 174629248 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated67437672 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 8931778899456 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed174675266971648 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8931691462656 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated67437672 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 174638920 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 107201248 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated174675291875328 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _runtime4503.289386684 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 8931778899456 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated29077875 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 8909918245002 trainer/global_step10149 $ _timestamp1.785799790466038e+09 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated183196132873728 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed69044501 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated39966884 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed174264336934912 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 107191578 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed39966882 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 107191578 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121ph2- _wandbruntime4503  _runtime4503_s  2loss/SRqqt1/assignment_loss1.0972120387165847 2loss/SRqqt2/assignment_loss0.9793311077064106 1loss/SRqqt1/detection_loss1.3403020673836379 /loss/FBt2/detection_loss2.3228437058361515 % _timestamp1.7857997904676943e+09 _runtime4503.290530944 _step811 0loss/FRt1/assignment_loss2.4620509850351433 3loss/FBt1/assignment_loss5.960460877455868e-08 /loss/FRt2/detection_loss2.2207611469227984 1loss/SRqqt2/detection_loss0.8745724216688615 /loss/FBt1/detection_loss0.6680777202981214 2loss/FBt2/assignment_loss0.011041614689905341 /loss/FRt1/detection_loss2.3126589200998606 %loss/total_loss16.47871279143354 0loss/FRt2/assignment_loss2.1898610034714574 epoch0 trainer/global_step10149 0loss/FRt2/assignment_loss2.1898610034714574 epoch0 trainer/global_step10149 2loss/SRqqt1/assignment_loss1.0972120387165847 2loss/SRqqt2/assignment_loss0.9793311077064106 1loss/SRqqt1/detection_loss1.3403020673836379 /loss/FBt2/detection_loss2.3228437058361515 % _timestamp1.7857997904676943e+09 _runtime4503.290530944 _step811 0loss/FRt1/assignment_loss2.4620509850351433 3loss/FBt1/assignment_loss5.960460877455868e-08 /loss/FRt2/detection_loss2.2207611469227984 1loss/SRqqt2/detection_loss0.8745724216688615 /loss/FBt1/detection_loss0.6680777202981214 2loss/FBt2/assignment_loss0.011041614689905341 /loss/FRt1/detection_loss2.3126589200998606 %loss/total_loss16.47871279143354V(:  V disk.drbd1356.out0 disk.drbd1510.in0 disk.drbd1226.out0 disk.drbd2111.in0 disk.drbd1817.in0 disk.drbd1965.in0 proc.cpu.threads29 disk.drbd1550.in0 disk.nvme0n1.in 4.64453125) disk./.usagePercent32.34416413141666 disk.drbd1965.out0 disk.sda.in0 disk.drbd1828.out0 disk.drbd2078.out0 disk.drbd1931.out0 disk.drbd2041.in0 disk.drbd1552.out0 disk.sda1.in0 disk.drbd1510.out0 disk.rbd1.out 18.0078125 disk.drbd1828.in0 disk.drbd1289.in0 disk.drbd1938.in0 disk.drbd1285.in0 disk.drbd1584.out0 disk.rbd1.in0 disk.drbd1673.in0 disk.drbd1550.out0 disk.drbd1289.out0* proc.memory.availableMB38505.80078125 disk.drbd2078.in0 disk.drbd1356.in0 disk.drbd2111.out0 disk.drbd1931.in0! proc.memory.rssMB 11260.40625$ disk./.usageGB578.3465042114258 disk.drbd1226.in0 disk.drbd2041.out0 disk.drbd1552.in0 cpu13.785183146712463 network.sent 9.278027e+06 network.recv562907" disk.nvme0n1.out 189.111328125 disk.drbd1938.out0 disk.rbd0.in0 disk.sda.out 124.65234375 disk.sda1.out0 disk.drbd1285.out0 disk.dm-0.out 189.111328125 disk.sda2.out 124.65234375 disk.rbd0.out0* proc.memory.percent17.182016372680664 disk.sda2.in0 disk.drbd1817.out0 disk.drbd1673.out0% memory_percent41.244810819625854 disk.drbd1584.in0 disk.drbd2118.in0 disk.drbd2118.out0 disk.dm-0.in 4.64453125.^9(: U gpu.0.gpu22.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts125.609' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent41.86966666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0! 2- _wandbruntime4525  _runtime4525z & lr-AdamW/pg10.0002628505299636369 & lr-AdamW/pg20.0002628505299636369 trainer/global_step10199 $ _timestamp1.785799812788323e+09 _runtime4525.611292207 _step812?HR & lr-AdamW/pg10.0002628505299636369 & lr-AdamW/pg20.0002628505299636369 trainer/global_step10199 $ _timestamp1.785799812788323e+09 _runtime4525.611292207 _step812Fr2- _wandbruntime4525  _runtime4525U O @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 trainer/global_step10199 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated175105525361664 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 8974895408128 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed175105508322304 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 107709057 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed69378192 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 _runtime4525.612508996 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current256 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated184736106944000 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed175518415694848 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated184080420769792 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated67763179 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated67763179 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9217666345472 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 107718713 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated170754319380832 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8952929235176 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current258 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated40159797 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed184735945074688 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated29218653 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 175472234 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 175481892 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated69378450 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated184080420769792 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed67763177 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8952843432052 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated175518440598528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed170754302341472 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8974807978496 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed184080316300800 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed179707145773524 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed184080316300800 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed67763177 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 107718713 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated175105525361664 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 8974807978496 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed175105508322304 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 8974895408128'O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed184735945074688 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated29218653 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 175472234 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 175481892 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated69378450 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated184080420769792 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed67763177 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 8952843432052 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated175518440598528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed170754302341472 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 8974807978496 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed184080316300800 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed179707145773524 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed184080316300800 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed67763177 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 107718713 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated175105525361664 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 8974807978496 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed175105508322304 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 8974895408128 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 trainer/global_step10199 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated175105525361664 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 8974895408128 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed175105508322304 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 107709057 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed69378192 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 _runtime4525.612508996 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current256 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated184736106944000 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed175518415694848 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated184080420769792 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated67763179 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated67763179 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9217666345472 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 107718713 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated170754319380832 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8952929235176 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current258 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated40159797 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0Xh2- _wandbruntime4525  _runtime4525vXQ&M S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed179724763419066 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current256 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 107729250 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 107719580 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated67769822 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed170771042513792 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed67769820 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed175122675085312 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed69384990 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed29221258 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 175489400 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 175499072 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed40163732 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated175535647715328 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated67769822 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated184098467159552 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated29221512 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated170771059553152 % _timestamp1.7857998131629586e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated184754212489216 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 9218432009728 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 107729250 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8975687598080 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed184098362683392 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated175122692124672 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed175122675085312 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed175535622811648 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 trainer/global_step10199 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 175489400 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed67769820 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed170771042513792 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed67769820 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed175122675085312 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed69384990 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed29221258 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 175489400 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E=DeviceStatsMoniXttor.on_train_batch_end/segment.small_pool.peak121 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 175499072 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed40163732 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated175535647715328 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated67769822 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated184098467159552 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated29221512 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated170771059553152 % _timestamp1.7857998131629586e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated184754212489216 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 9218432009728 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 107729250 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 8975687598080 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed184098362683392 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated175122692124672 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed175122675085312 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed175535622811648 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 trainer/global_step10199 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 175489400 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed67769820 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 _runtime4547.829608725 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated185641629441536 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated176378796438528 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 9018891543552 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed185641471766528 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed171591310957472 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 9262700231680 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed184982650586624 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated175963863512064 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed175963846472704 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated68095329 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed68095327 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed68095327 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed69719315 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current258 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated184982755055616 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed184982650586624 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9262833003008 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated180588145695516 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 trainer/global_step10249 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 176332386 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated68095329 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8996817698684 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 9018891543552 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9018804113920 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 108246715 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed175963846472704 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 176342044 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251zd,'O N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 176342044 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 _runtime4547.829608725 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated185641629441536 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated176378796438528 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 9018891543552 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed185641471766528 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed171591310957472 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 9262700231680 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed184982650586624 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated175963863512064 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed175963846472704 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated68095329 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed68095327 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed68095327 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed69719315 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current258 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated184982755055616 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed184982650586624 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9262833003008 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated180588145695516 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 trainer/global_step10249 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 176332386 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated68095329 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 8996817698684 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 9018891543552 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9018804113920 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 108246715 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed175963846472704 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200s%;2- _wandbruntime4548  _runtime4548 SM FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated185000801445376 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 SEDeviceStatsMonit*@!or.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 176359224 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed69726096 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current232 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 176359224 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated68101972 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9019771170304 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9019771170304 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 _step818 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed68101970 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed175981013235712 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed185659592195584 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed40360582 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9019683733504 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated69726330 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 108247582 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed175981013235712 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed185000696969216 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 140890624 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 _runtime4548.204319287 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed68101970 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed180605660498574 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated185000801445376 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 115986944 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 108257252 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 trainer/global_step10249 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated176396003555328 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392&ڰ&M BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed68101970 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed180605660498574 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated185000801445376 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 115986944 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 108257252 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 trainer/global_step10249 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated176396003555328 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated185000801445376 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 176359224 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed69726096 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current232 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 176359224 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated68101972 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9019771170304 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9019771170304 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 _step818 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed68101970 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed175981013235712 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed185659592195584 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed40360582 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9019683733504 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated69726330 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 108247582 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed175981013235712 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed185000696969216 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 140890624 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 _runtime4548.204319287 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856b[2- _wandbruntime4548  _runtime4548x  /loss/FRt2/assignment_loss2.339753270149231 2loss/SRqqt1/assignment_loss0.9223948855139316 2loss/SRqqt2/assignment_loss1.0628365880715192 3loss/FBt1/assignment_loss3.214382482749186e-05 0loss/SRqqt2/detection_loss1.117800440947364 trainer/global_step10249 0loss/FRt1/assignment_loss3.1471901359692427 3loss/FBt2/assignment_loss0.0006547102497833635 /loss/FRt1/detection_loss2.7146543197228876 1loss/SRqqt1/detection_loss1.7363814900442958 .loss/FBt2/detection_loss2.572192408431035 &loss/total_loss18.783120815365756 epoch0 _step819 /loss/FBt1/detection_loss0.8033583433904324 % _timestamp1.7857998353826077e+09 /loss/FRt2/detection_loss2.3658720790512033 _runtime4548.205433908Ǜ* /loss/FRt2/detection_loss2.3658720790512033 _runtime4548.205433908 /loss/FRt2/assignment_loss2.339753270149231 2loss/SRqqt1/assignment_loss0.9223948855139316 2loss/SRqqt2/assignment_loss1.0628365880715192 3loss/FBt1/assignment_loss3.214382482749186e-05 0loss/SRqqt2/detection_loss1.117800440947364 trainer/global_step10249 0loss/FRt1/assignment_loss3.1471901359692427 3loss/FBt2/assignment_loss0.0006547102497833635 /loss/FRt1/detection_loss2.7146543197228876 1loss/SRqqt1/detection_loss1.7363814900442958 .loss/FBt2/detection_loss2.572192408431035 &loss/total_loss18.783120815365756 epoch0 _step819 /loss/FBt1/detection_loss0.8033583433904324 % _timestamp1.7857998353826077e+09>aҐ(: U gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts126.339' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent42.113 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors03щW(:  U disk.drbd1550.in0 disk.drbd1226.out0 disk.drbd1356.out0 disk.sda2.out 125.65234375 disk.drbd1673.in0 disk.drbd1931.out0 disk.rbd0.in0 disk.sda1.in0 disk.drbd1965.in0 disk.drbd1828.in0 disk.drbd1285.out0 disk.nvme0n1.in 4.64453125 disk.drbd2111.in0* proc.memory.percent17.182016372680664 disk.drbd1673.out0 disk.drbd1552.in0 disk.drbd1289.in0 proc.cpu.threads29 disk.drbd1510.in0 disk.drbd1938.in0 disk.drbd1817.in0 disk.drbd1817.out0 disk.drbd1965.out0 disk.drbd1550.out0 disk.dm-0.in 4.64453125 disk.drbd2041.out0 disk.drbd1289.out0 disk.drbd2041.in0 disk.drbd2118.out0 disk.sda2.in0 disk.sda.in0 disk.drbd1285.in0 disk.drbd1584.in0 disk.drbd1356.in0$ memory_percent41.24539494514465! proc.memory.rssMB 11260.40625 disk.rbd0.out0 disk.sda1.out0 disk.drbd2111.out0 network.recv567228 disk.drbd1584.out0 disk.drbd2118.in0) disk./.usagePercent32.34424925321516 disk.rbd1.out18.40625 disk.drbd1552.out0 disk.sda.out 125.65234375 disk.drbd2078.in0 disk.rbd1.in0 disk.drbd1931.in0 disk.drbd1938.out0 disk.drbd1226.in0$ disk./.usageGB578.3480262756348 disk.drbd1510.out0 cpu13.787653816139144! disk.dm-0.out190.00634765625 disk.drbd2078.out0$ disk.nvme0n1.out190.00634765625* proc.memory.availableMB38505.41796875 network.sent 9.371061e+06 disk.drbd1828.out0^MN2- _wandbruntime4570  _runtime4570ShE trainer/global_step10299 % _timestamp1.7857998576582625e+09 _runtime4570.481226951 _step820 ' lr-AdamW/pg10.00026542774861216753 ' lr-AdamW/pg20.00026542774861216753, ' lr-AdamW/pg20.00026542774861216753 trainer/global_step10299 % _timestamp1.7857998576582625e+09 _runtime4570.481226951 _step820 ' lr-AdamW/pg10.00026542774861216753ܹ2- _wandbruntime4570  _runtime4570{)W'O @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed29505642 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed186547007808512 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 177192538 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed177239127374848 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 9307880433664 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9040620359068 trainer/global_step10299 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated68427479 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed185884984872448 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated68427479 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed70059137 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated176822201662464 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 9062887678976 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 9062800249344 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 145354240 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 9040706162192 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 _runtime4570.485666372 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 177202196 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 108765061 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 108774717 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 108765061 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated185885089341440 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated176822201662464 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed68427477 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 9062887678976 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 170257920 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9062800249344 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/Xۀallocation.large_pool.freed68427477 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated29505924DS'O LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated68427479 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed185884984872448 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated68427479 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed70059137 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated176822201662464 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 9062887678976 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 9062800249344 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 145354240 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 9040706162192 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 _runtime4570.485666372 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 177202196 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 108765061 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 108774717 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 108765061 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated185885089341440 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated176822201662464 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed68427477 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 9062887678976 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 170257920 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9062800249344 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed68427477 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated29505924 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed29505642 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed186547007808512 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 177192538 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed177239127374848 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 9307880433664 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9040620359068 trainer/global_step10299 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392Λ2- _wandbruntime4570  _runtime4570w%0&M E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9063767305728 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated172445076785152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated185903135731200 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9063679868928 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 177219376 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated70066336 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 9308918352896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9063679868928 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current260 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated68434122 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 108775584 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated181486660420678 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 177219376 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 108785254 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 _runtime4570.854461126 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated176839368425472 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed70066074 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 108775584 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed176839351386112 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed177256334491648 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 % _timestamp1.7857998580310977e+09 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed29508642 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed181486557578082 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 177209704 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed68434120 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current262 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated29508902 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed68434120 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed172445059745792 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 trainer/global_step10299 _step822 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9041583635526 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated68434122 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 108775584 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated181486660420678 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 177219376 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 108785254 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 _runtime4570.854461126 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated176839368425472 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed70066074 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 108775584 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed176839351386112 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed177256334491648 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 % _timestamp1.7857998580310977e+09 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed29508642 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed181486557578082 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 177209704 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed68434120 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current262 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated29508902 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed68434120 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed172445059745792 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 trainer/global_step10299 _step822 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9041583635526 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 177219376 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated70066336 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 9308918352896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free04W0*2- _wandbruntime4570  _runtime4570=  /loss/FRt1/detection_loss2.6823915850806546 trainer/global_step10299 $ _timestamp1.785799858032869e+09 0loss/FRt2/assignment_loss2.2612117332346897 3loss/FBt2/assignment_loss0.0010764240942062742 &loss/total_loss17.761219648538944 _runtime4570.855728813 2loss/SRqqt1/assignment_loss0.8939361083191923 3loss/FBt1/assignment_loss7.050908154988227e-05 -loss/FRt2/detection_loss2.17493750258012 1loss/SRqqt1/detection_loss1.6019509663736378 1loss/SRqqt2/detection_loss1.0164648873421054 /loss/FBt1/detection_loss0.8840729616582393 epoch0 _step823 0loss/FRt1/assignment_loss2.6882369138977746 1loss/SRqqt2/assignment_loss0.995397372731801 /loss/FBt2/detection_loss2.5614726841449738ӱ 2loss/SRqqt1/assignment_loss0.8939361083191923 3loss/FBt1/assignment_loss7.050908154988227e-05 -loss/FRt2/detection_loss2.17493750258012 1loss/SRqqt1/detection_loss1.6019509663736378 1loss/SRqqt2/detection_loss1.0164648873421054 /loss/FBt1/detection_loss0.8840729616582393 epoch0 _step823 0loss/FRt1/assignment_loss2.6882369138977746 1loss/SRqqt2/assignment_loss0.995397372731801 /loss/FBt2/detection_loss2.5614726841449738 /loss/FRt1/detection_loss2.6823915850806546 trainer/global_step10299 $ _timestamp1.785799858032869e+09 0loss/FRt2/assignment_loss2.2612117332346897 3loss/FBt2/assignment_loss0.0010764240942062742 &loss/total_loss17.761219648538944 _runtime4570.8557288135(: ⑀U gpu.0.gpu37.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts96.469' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent32.15633333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors09/(:  U disk.drbd1673.out0 disk.sda1.out0 disk.drbd1510.out0 disk.drbd1226.in0 disk.drbd2111.in0 disk.drbd2111.out0 disk.drbd1550.in0 disk.drbd1289.out0 disk.sda.out 125.89453125 disk.drbd1510.in0 disk.drbd2041.out0 disk.drbd1552.in0 disk.drbd1828.out0 disk.sda1.in0 disk.nvme0n1.in\a 4.64453125 disk.rbd1.out18.40625 disk.drbd1931.in0 disk.rbd0.in0$ disk.nvme0n1.out190.15869140625$ disk./.usageGB578.3480567932129 disk.rbd1.in0 disk.drbd1552.out0 disk.drbd1285.out0 disk.drbd2118.in0 disk.drbd2118.out0 disk.drbd1356.out0 disk.sda2.out 125.89453125 disk.drbd1285.in0 disk.drbd1226.out0 disk.sda2.in0 proc.cpu.threads29 disk.drbd1550.out0 disk.sda.in0 disk.drbd1584.in0 disk.drbd1817.in0 disk.drbd1938.in0) disk./.usagePercent32.34425095991788 disk.drbd1673.in0! proc.memory.rssMB 11260.40625 disk.drbd2041.in0$ memory_percent41.24566912651062 network.sent 9.416025e+06 disk.drbd1965.in0 disk.drbd1938.out0! disk.dm-0.out190.15869140625 disk.drbd2078.in0* proc.memory.percent17.182016372680664 disk.drbd1965.out0 disk.drbd1289.in0 disk.rbd0.out0 disk.drbd1817.out0 disk.dm-0.in 4.64453125 disk.drbd2078.out0 disk.drbd1356.in0* proc.memory.availableMB38505.23828125 cpu13.787101425189457 disk.drbd1931.out0 disk.drbd1828.in0 network.recv568666 disk.drbd1584.out0;(: U gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts114.813' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent38.271 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0]W(:  áU disk.drbd1289.in0 disk.rbd0.in0 disk.drbd2118.in0 disk.drbd1817.in0 disk.drbd1965.in0! disk.dm-0.out190.69287109375 disk.drbd2118.out0 disk.drbd2111.in0 disk.drbd1817.out0 disk.drbd1938.out0 disk.drbd1285.out0 disk.drbd1550.out0 disk.drbd1938.in0 disk.sda.out 126.18359375 disk.drbd1510.in0 disk.drbd1226.out0 disk.drbd2078.in0 disk.drbd1584.in0 disk.sda2.out 126.18359375 disk.rbd1.out18.46875 disk.drbd1289.out0 network.sent 9.461079e+06 disk.drbd1673.out0 disk.drbd1552.in0 disk.drbd1552.out0 disk.drbd1828.in0* proc.memory.availableMB38505.44140625 network.recv570415 disk.drbd2041.out0 disk.drbd1550.in0 disk.drbd1584.out0 disk.dm-0.in 4.64453125* proc.memory.percent17.182016372680664 disk.drbd1931.in0 disk.drbd1965.out0 disk.drbd1828.out0 disk.sda1.in0 disk.drbd2078.out0 disk.drbd2041.in0$ disk.nvme0n1.out190.69287109375 disk.drbd1931.out0 disk.drbd1285.in0 disk.drbd1510.out0 disk.drbd1356.out0! proc.memory.rssMB 11260.40625 proc.cpu.threads29$ disk./.usageGB578.3480567932129 disk.sda2.in0 disk.rbd0.out0) disk./.usagePercent32.34425095991788 disk.rbd1.in0 disk.drbd1673.in0 disk.sda.in0 disk.nvme0n1.in 4.64453125 disk.drbd1356.in0 disk.drbd1226.in0 disk.drbd2111.out0 disk.sda1.out0$ memory_percent41.24535918235779 cpu13.786336808033838s2- _wandbruntime4592  _runtime4592i4 & lr-AdamW/pg10.0002667163579364328 & lr-AdamW/pg20.0002667163579364328 trainer/global_step10349 % _timestamp1.7857998795031166e+09 _runtime4592.326071311 _step824|08 & lr-AdamW/pg10.0002667163579364328 & lr-AdamW/pg20.0002667163579364328 trainer/global_step10349 % _timestamp1.7857998795031166e+09 _runtime4592.326071311 _step824?2- _wandbruntime4592  _runtime4592d'O WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9106796384768 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated182349939854532 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 % _timestamp1.7857998795040674e+09 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated29649043 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed178099483214848 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed177680522773504 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed182349837012048 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed68759627 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated70399390 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed186787319158272 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated187452697537024 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated178099508118528 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed40750345 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated186787423627264 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 9106883814400 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current246 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed70399144 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9353189418496 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated40750347 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 9084594625700F'O WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed68759627 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9084508822576 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 _runtime4592.327264136 _step825 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 9106796384768 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current244 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9353189418496 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated40750347 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 9084594625700 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9106796384768 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated182349939854532 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 % _timestamp1.7857998795040674e+09 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated29649043 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed178099483214848 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed177680522773504 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed182349837012048 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed68759627 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated70399390 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed186787319158272 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated187452697537024 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated178099508118528 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed40750345 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated186787423627264 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 9106883814400 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current246 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed70399144 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 JϘ!M @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9107676004352 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 109313256 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated40754284 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed173282068361792 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 178069856 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed70405930 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed40754282 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated68766272 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9107763441152 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 109303586 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed178116690331648 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated173282085401152 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 9085386295798 % _timestamp1.7857998798701131e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 9354093786112 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 9353956827648 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated186805470017024 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated70406187 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated68766272 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 109313256 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated29651903 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9085472099034 _step826 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9107763441152 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current257 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 178079528 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 178069856 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 178079528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9107676004352 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed187470647159296 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 trainer/global_step10349 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated177697706575872 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed186805365540864 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed29651648 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed177697689536512 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current255 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated177697706575872 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocatϪ(ed187470809021440 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated178116715235328 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed177697689536512 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 109303586 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0f6SV&M T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated186805470017024 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated70406187 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated68766272 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 109313256 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated29651903 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9085472099034 _step826 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9107763441152 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current257 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 178079528 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 178069856 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 178079528 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9107676004352 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed187470647159296 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 trainer/global_step10349 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated177697706575872 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed186805365540864 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed29651648 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed177697689536512 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current255 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated177697706575872 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated187470809021440 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated178116715235328 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed177697689536512 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 109303586 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9107676004352 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 109313256 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated40754284 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed173282068361792 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 178069856 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed70405930 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed40754282 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated68766272 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9107763441152 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 109303586 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed178116690331648 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated173282085401152 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 9085386295798 % _timestamp1.7857998798701131e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 9354093786112 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 9353956827648 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236l 12- _wandbruntime4592  _runtime4592JO 3loss/FBt1/assignment_loss0.0034416781272168517 /loss/FBt1/detection_loss1.0705445990315638 /loss/FBt2/detection_loss1.5956919243415961 trainer/global_step10349 _runtime4592.694561581 .loss/FRt1/assignment_loss2.48743797974153 .loss/FRt2/assignment_loss2.18875037248318 2loss/SRqqt1/assignment_loss0.8704228241670322 0loss/SRqqt1/detection_loss1.824532188158086 /loss/FRt2/detection_loss2.5132864917126985 1loss/SRqqt2/detection_loss1.0754508367169273 &loss/total_loss17.344034974271743 _step827 0loss/FBt2/assignment_loss0.1844329283787654 -loss/FRt1/detection_loss2.55877668207342 epoch0 % _timestamp1.7857998798717396e+09 2loss/SRqqt2/assignment_loss0.9712664693397246/I /loss/FRt2/detection_loss2.5132864917126985 1loss/SRqqt2/detection_loss1.0754508367169273 &loss/total_loss17.344034974271743 _step827 0loss/FBt2/assignment_loss0.1844329283787654 -loss/FRt1/detection_loss2.55877668207342 epoch0 % _timestamp1.7857998798717396e+09 2loss/SRqqt2/assignment_loss0.9712664693397246 3loss/FBt1/assignment_loss0.0034416781272168517 /loss/FBt1/detection_loss1.0705445990315638 /loss/FBt2/detection_loss1.5956919243415961 trainer/global_step10349 _runtime4592.694561581 .loss/FRt1/assignment_loss2.48743797974153 .loss/FRt2/assignment_loss2.18875037248318 2loss/SRqqt1/assignment_loss0.8704228241670322 0loss/SRqqt1/detection_loss1.824532188158086g6T(:  ӯU* proc.memory.percent17.182016372680664 disk.rbd0.in0 disk.drbd1510.out0) disk./.usagePercent32.34422685274186 disk.drbd1817.out0 disk.drbd1965.out0 disk.sda.in0 disk.drbd2118.out0 disk.drbd1931.out0 disk.sda.out 126.54296875 disk.sda1.out0 disk.dm-0.in 4.64453125 disk.drbd2111.in0 disk.drbd2111.out0 disk.drbd1938.out0 disk.drbd1226.in0 proc.cpu.threads29 disk.drbd1673.out0 disk.drbd1828.in0 disk.sda1.in0! disk.dm-0.out190.85693359375! proc.memory.rssMB 11260.40625 disk.drbd1673.in0$ disk./.usageGB578.3476257324219 disk.drbd1965.in0 disk.drbd1285.in0 disk.nvme0n1.in 4.64453125 network.sent 9.464153e+06 network.recv571313 disk.rbd1.in0 disk.drbd1550.in0 disk.drbd1226.out0 disk.drbd2078.in0 disk.drbd2078.out0 disk.drbd2118.in0 disk.drbd1828.out0 disk.drbd1938.in0$ disk.nvme0n1.out190.85693359375 disk.sda2.out 126.54296875' proc.memory.availableMB 38505.90625 disk.drbd1289.in0 disk.drbd1289.out0 disk.drbd1817.in0 disk.rbd1.out18.46875 disk.rbd0.out0 disk.drbd1584.out0 disk.drbd2041.out0 disk.drbd1510.in0 disk.drbd1931.in0 disk.drbd2041.in0 disk.drbd1285.out0 disk.drbd1356.out0 cpu13.785630929591324 disk.drbd1552.out0 disk.drbd1552.in0 disk.drbd1550.out0 disk.drbd1356.in0$ memory_percent41.24464988708496 disk.drbd1584.in0 disk.sda2.in0ɠq>(: V gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts127.587' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent42.529 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0l2- _wandbruntime4614  _runtime4614M trainer/global_step10399 $ _timestamp1.785799901495674e+09 _runtime4614.320703173 _step828 & lr-AdamW/pg10.0002680049672606981 & lr-AdamW/pg20.0002680049672606981ϵg trainer/global_step10399 $ _timestamp1.785799901495674e+09 _runtime4614.320703173 _step828 & lr-AdamW/pg10.0002680049672606981 & lr-AdamW/pg20.00026800496726069812- _wandbruntime4614  _runtime4614VF'O PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed69091777 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 _runtime4614.320801769 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 178922500 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed187689653444096 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed178538860923904 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated29792661 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated174102353844832 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated69091779 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9128397286084 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed174102336805472 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 109830721 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 109830721 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated188358235782144 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 % _timestamp1.7857999014963512e+09 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9398371823616 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 _step829 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed69091777 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed70739588 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current268 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 9398234857984 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated178959863958528 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated178538877963264 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated187689757913088 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed29792393 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 178912842 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated187689757913088 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed178538860923904 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed178959839054848 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 9150879949824 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated40947197 BO XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated29792661 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated174102353844832 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated69091779 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9128397286084 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed174102336805472 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 109830721 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 109830721 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated188358235782144 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 % _timestamp1.7857999014963512e+09 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9398371823616 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 _step829 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed69091777 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed70739588 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current268 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 9398234857984 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated178959863958528 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated178538877963264 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated187689757913088 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed29792393 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 178912842 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small`r _pool.peak 250523136 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated187689757913088 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed178538860923904 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed178959839054848 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 9150879949824 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated40947197 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed187689653444096 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed178538860923904 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936m2- _wandbruntime4614  _runtime4614DCr&M PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed178556027686912 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9151759576576 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated69098422 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed70746361 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated178556044726272 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed178556027686912 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9151672139776 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed188376201929728 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 trainer/global_step10399 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current246 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed29795229 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 % _timestamp1.7857999018681664e+09 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated174119094017152 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated187707804302848 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated187707804302848 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated188376351208960 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 124375552 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated40951134 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9151759576576 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9151672139776 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed183248351737098 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed178977046171648 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9129360562542 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 178939680 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated178556044726272 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 178939680 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated29795473 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated69098422 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 9129274759306 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed69098420 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 178930008 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 149279232 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 109841258 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated70746607 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 _runtime4614.691515483 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated178977071075328 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 9399280133632 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated183248454579694 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed174119076977792 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0n f&M ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated174119094017152 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated187707804302848 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated187707804302848 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated188376351208960 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 124375552 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated40951134 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9151759576576 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9151672139776 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed183248351737098 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed178977046171648 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9129360562542 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 178939680 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated178556044726272 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 178939680 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated29795473 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated69098422 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 9129274759306 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed69098420 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 178930008 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 149279232 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 109841258 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated70746607 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 _runtime4614.691515483 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated178977071075328 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 9399280133632 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated183248454579694 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed174119076977792 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed178556027686912 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9151759576576 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated69098422 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed70746361 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated178556044726272 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed178556027686912 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9151672139776 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed188376201929728 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 trainer/global_step10399 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current246 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed29795229 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 % _timestamp1.7857999018681664e+09/&2- _wandbruntime4614  _runtime4614H  _runtime 4614.69319893 .loss/FRt2/detection_loss2.385313917096559 0loss/SRqqt1/detection_loss1.868965139294543 /loss/FBt2/detection_loss1.8077701595094469 trainer/global_step10399 0loss/FRt1/assignment_loss2.8645920982727637 -loss/FRt1/detection_loss2.49251873594719 epoch0 _step831 0loss/FRt2/assignment_loss2.1918836824305648 4loss/FBt1/assignment_loss2.6523519933707718e-06 /loss/FBt1/detection_loss1.4005089400961879 &loss/total_loss17.468013258125033 2loss/SRqqt1/assignment_loss0.6079854447758053 2loss/SRqqt2/assignment_loss0.8294714879287456 4loss/FBt2/assignment_loss1.0596378767028607e-07 0loss/SRqqt2/detection_loss1.019000894457445 % _timestamp1.7857999018703573e+09h4y epoch0 _step831 0loss/FRt2/assignment_loss2.1918836824305648 4loss/FBt1/assignment_loss2.6523519933707718e-06 /loss/FBt1/detection_loss1.4005089400961879 &loss/total_loss17.468013258125033 2loss/SRqqt1/assignment_loss0.6079854447758053 2loss/SRqqt2/assignment_loss0.8294714879287456 4loss/FBt2/assignment_loss1.0596378767028607e-07 0loss/SRqqt2/detection_loss1.019000894457445 % _timestamp1.7857999018703573e+09 _runtime 4614.69319893 .loss/FRt2/detection_loss2.385313917096559 0loss/SRqqt1/detection_loss1.868965139294543 /loss/FBt2/detection_loss1.8077701595094469 trainer/global_step10399 0loss/FRt1/assignment_loss2.8645920982727637 -loss/FRt1/detection_loss2.49251873594719̩,(: U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts132.864' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent44.288 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0/\(:  U disk.drbd1552.out0 disk.drbd1550.out0 disk.drbd1938.out0 disk.drbd1285.in0 disk.sda2.in0 proc.cpu.threads29 disk.drbd1550.in0 disk.drbd1584.out0 network.sent 9.509157e+06 disk.rbd0.in0 disk.dm-0.in 4.64453125 disk.nvme0n1.in 4.64453125 disk.drbd2111.out0 disk.drbd1965.out0 disk.drbd2041.out0 disk.sda.in0! proc.memory.rssMB 11260.40625 disk.drbd1931.out0 disk.drbd1356.in0 disk.drbd1828.in0 disk.sda.out 126.83203125! disk.dm-0.out191.24755859375 disk.drbd1226.in0 disk.drbd2111.in0 disk.drbd1931.in0 disk.sda1.out0 disk.drbd1510.in0 disk.drbd2118.out0 disk.drbd1673.out0* proc.memory.availableMB38505.12109375 disk.drbd1552.in0 disk.drbd1226.out0 disk.drbd2078.in0 disk.drbd1584.in0% memory_percent41.245847940444946 cpu13.784903507645595 disk.drbd2041.in0 disk.drbd1289.out0 disk.drbd1938.in0 disk.drbd1356.out0* proc.memory.percent17.182016372680664 disk.sda1.in0 disk.drbd2118.in0 disk.drbd1817.in0 disk.drbd1965.in0$ disk./.usageGB578.3476104736328 disk.drbd1289.in0 disk.drbd1285.out0 disk.drbd1510.out0 disk.drbd2078.out0 network.recv573121 disk.drbd1828.out0 disk.rbd1.out 18.6806640625 disk.rbd0.out0$ disk.nvme0n1.out191.24755859375 disk.drbd1817.out0( disk./.usagePercent32.3442259993905 disk.sda2.out 126.83203125 disk.rbd1.in0 disk.drbd1673.in0l(: шU gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts129.477' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent43.159000000000006 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors07^(:  U network.sent 9.554071e+06 disk.rbd0.in0 disk.drbd1285.out0 disk.drbd1226.out0* disk./.usagePercent32.344221945971526 disk.drbd1965.in0 disk.drbd1931.out0 disk.drbd1285.in0 disk.drbd2118.in0 disk.sda2.out 127.24609375 disk.drbd1550.out0 disk.drbd1938.in0 disk.drbd1510.out0% memory_percent41.245847940444946 disk.rbd1.out 18.6806640625 disk.drbd1965.out0 disk.sda.in0 network.recv574692$ disk./.usageGB578.3475379943848 disk.drbd1673.in0 disk.drbd2078.in0 disk.drbd1584.in0 disk.drbd1552.in0 disk.drbd2118.out0 disk.drbd1510.in0 disk.drbd1356.in0 disk.drbd1938.out0* proc.memory.percent17.182016372680664 cpu13.784180451869386 disk.sda1.in0 disk.drbd1817.in0 disk.drbd2041.out0 disk.drbd1552.out0 disk.drbd1828.in0 disk.drbd1550.in0 disk.sda.out 127.24609375! disk.dm-0.out191.43896484375 disk.drbd1226.in0 disk.sda2.in0 disk.drbd1817.out0 disk.drbd1673.out0 disk.rbd0.out0 disk.dm-0.in 4.64453125! proc.memory.rssMB 11260.40625 disk.drbd2111.out0 disk.drbd2041.in0 disk.drbd1289.in0 disk.sda1.out0 disk.drbd1584.out0 disk.nvme0n1.in 4.64453125$ disk.nvme0n1.out191.43896484375 disk.drbd1931.in0 disk.drbd1356.out0 proc.cpu.threads29 disk.drbd1828.out0 disk.drbd2078.out0* proc.memory.availableMB38505.12109375 disk.drbd2111.in0 disk.rbd1.in0 disk.drbd1289.out0D2- _wandbruntime4636  _runtime4636|i % _timestamp1.7857999238956192e+09 _runtime4636.719750695 _step832 & lr-AdamW/pg10.0002692935765849634 & lr-AdamW/pg20.0002692935765849634 trainer/global_step10449 V _step832 & lr-AdamW/pg10.0002692935765849634 & lr-AdamW/pg20.0002692935765849634 trainer/global_step10449 % _timestamp1.7857999238956192e+09 _runtime4636.719750695&w<2- _wandbruntime4636  _runtime4636 s O N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 179782652 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current252 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed29935835 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 110358723 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 9194876085248 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated41144047 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated174939362460832 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9194788655616 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed179820194894848 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed189263557117440 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9443486605312 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated69423929 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated29936087 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed179397199074304 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 LDDeviceStatsMyq(;onitor.on_train_batch_start/segment.small_pool.allocated121 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed188591987729920 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed174939345421472 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current254 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 110349067 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 110349067 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated189263706403840 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 124382720 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed41144045 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated184111734013548 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed179397199074304 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 % _timestamp1.7857999238977115e+09 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9172285749592 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 179772994 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed188591987729920 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated179820219798528 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated69423929 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated188592092198912 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 9194788655616 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed71079880 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 trainer/global_step10449 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 9194876085248 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 _step833 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 149286400 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 9172371552716 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed69423927 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated179397216113664 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed188591987729920 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed174939345421472 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current254 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 110349067 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 110349067 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated189263706403840 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 124382720 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed41144045 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated184111734013548 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed179397199074304 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 % _timestamp1.7857999238977115e+09 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9172285749592 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 179772994 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed188591987729920 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated179820219798528 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated69423929 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated188592092198912 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 9194788655616 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed71079880 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 trainer/global_step10449 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 9194876085248 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 _step833 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 149286400 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 9172371552716 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed69423927 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated179397216113664 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed189263557117440 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9443486605312 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated69423929 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated29936087 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed179397199074304 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current874296322- _wandbruntime4637  _runtime46370&M NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed69430570 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated71086885 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated41147984 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed179414365837312 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed71086637 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated69430572 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9195668275200 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed41147982 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 9444266810368 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated69430572 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed29938655 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9173249026050 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated188610138588672 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current248 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 179790160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated179414382876672 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 179799832 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed179837402011648 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed179414365837312 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 9173163222814 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated189281822295552 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed189281668822016 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed188610034112512 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 179790160 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 110369260 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 _runtime4637.096919753 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9195755712000 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 110359590 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 110359590 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated179837426915328 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed69430570 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated184129351659202 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 _step834 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 179799832 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed184129248816606 % _timestamp1.7857999242735753e+09 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated174956102633152 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current246 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 9444395380224 trainer/global_step10449 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9195668275200M UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated71086885 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated41147984 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed179414365837312 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed71086637 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated69430572 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9195668275200 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed41147982 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 9444266810368 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated69430572 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed29938655 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9173249026050 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated188610138588672 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current248 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 179790160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated179414382876672 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 179799832 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed179837402011648 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed179414365837312 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 9173163222814 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated189281822295552 XDDeviceStatsMonitor.on_train_batch_end/inactive_spliM_ 2t_bytes.all.freed189281668822016 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed188610034112512 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 179790160 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 110369260 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 _runtime4637.096919753 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9195755712000 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 110359590 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 110359590 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated179837426915328 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed69430570 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated184129351659202 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 _step834 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 179799832 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed184129248816606 % _timestamp1.7857999242735753e+09 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated174956102633152 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current246 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 9444395380224 trainer/global_step10449 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9195668275200 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed69430570 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0IJ2- _wandbruntime4637  _runtime4637 .loss/FRt1/detection_loss2.517832748955343 .loss/FBt2/detection_loss3.109997066576034 epoch0 $ _timestamp1.785799924275241e+09 _runtime4637.098064272 /loss/FRt1/assignment_loss3.092322378623776 /loss/FRt2/detection_loss2.4486418657186553 .loss/FBt1/detection_loss0.918505668051933 trainer/global_step10449 2loss/SRqqt2/assignment_loss0.7960311221581897 0loss/SRqqt2/detection_loss1.081316282710349 &loss/total_loss18.714106611056287 0loss/FRt2/assignment_loss2.3768147600860132 2loss/SRqqt1/assignment_loss0.7995616496354342 0loss/SRqqt1/detection_loss1.554831630960107 _step835 2loss/FBt1/assignment_loss0.017941916888355985 3loss/FBt2/assignment_loss0.0003095206920960436K  .loss/FRt1/detection_loss2.517832748955343 .loss/FBt2/detection_loss3.109997066576034 epoch0 $ _timestamp1.785799924275241e+09 _runtime4637.098064272 /loss/FRt1/assignment_loss3.092322378623776 /loss/FRt2/detection_loss2.4486418657186553 .loss/FBt1/detection_loss0.918505668051933 trainer/global_step10449 2loss/SRqqt2/assignment_loss0.7960311221581897 0loss/SRqqt2/detection_loss1.081316282710349 &loss/total_loss18.714106611056287 0loss/FRt2/assignment_loss2.3768147600860132 2loss/SRqqt1/assignment_loss0.7995616496354342 0loss/SRqqt1/detection_loss1.554831630960107 _step835 2loss/FBt1/assignment_loss0.017941916888355985 3loss/FBt2/assignment_loss0.0003095206920960436"(: U gpu.0.gpu34.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts127.13' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent42.376666666666665 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0#oS(:  U disk.drbd1510.in0 disk.drbd1356.out0 disk.rbd1.out 18.7431640625 disk.drbd1552.out0 disk.sda.in0 network.recv575695$ disk./.usageGB578.3476333618164 disk.drbd2111.out0 disk.drbd1965.out0 disk.drbd1550.in0 disk.drbd1938.in0 disk.rbd0.out0 disk.drbd1226.in0' proc.memory.availableMB 38505.03125! proc.memory.rssMB 11260.40625 disk.drbd2118.in0 disk.drbd1817.in0 disk.drbd1673.out0 proc.cpu.threads29 disk.drbd2041.in0 disk.drbd1285.in0 disk.drbd2078.out0 disk.nvme0n1.in 4.64453125 disk.drbd1828.out0 disk.drbd1938.out0 disk.drbd1285.out0 disk.drbd2111.in0 disk.drbd1828.in0 disk.sda1.out0 disk.drbd1510.out0 disk.drbd2078.in0 disk.drbd1817.out0 disk.drbd1931.in0 disk.rbd0.in0* proc.memory.percent17.182016372680664 cpu13.78359597880242 disk.drbd1584.out0 disk.drbd1289.in0 disk.sda.out 127.734375 disk.dm-0.in 4.64453125 disk.drbd2118.out0 disk.drbd1965.in0 disk.drbd1550.out0 disk.dm-0.out191.7626953125* disk./.usagePercent32.344227279417545# disk.nvme0n1.out191.7626953125 disk.drbd1289.out0 disk.sda2.in0 disk.drbd2041.out0 disk.drbd1552.in0 disk.drbd1226.out0 network.sent 9.557201e+06 disk.drbd1584.in0 disk.rbd1.in0 disk.drbd1356.in0 disk.sda2.out 127.734375$ memory_percent41.24598503112793 disk.drbd1673.in0 disk.drbd1931.out0 disk.sda1.in02- _wandbruntime4658  _runtime4658-Dr % _timestamp1.7857999459047525e+09 _runtime 4658.72771804 _step836 & lr-AdamW/pg10.0002705821859092287 & lr-AdamW/pg20.0002705821859092287 trainer/global_step10499K trainer/global_step10499 % _timestamp1.7857999459047525e+09 _runtime 4658.72771804 _step836 & lr-AdamW/pg10.0002705821859092287 & lr-AdamW/pg20.00027058218590922877?2- _wandbruntime4658  _runtime4658'O SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated41340897 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed175776354037472 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated71420487 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated189494426484736 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed190169075411456 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9216174213100 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 110886725 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated180255554264064 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed189494322015744 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed180680550734848 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 % _timestamp1.7857999459056807e+09 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated175776371076832 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed184992528250572 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9238784791040 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 180633146 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed180255537224704 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 180642804 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current249 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated184992631093056 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 9238872220672 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated69756079 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 110886725 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed189494322015744 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated30079590 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed30079343 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 9488524676608 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed180255537224704 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 _runtime4658.728889907 _step837 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated180680575638528 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed69756077 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed71420238 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current247 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9488657447936 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated189494426484736 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed189494322015744 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated30079590 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed30079343 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 9488524676608 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed180255537224704 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 _runtime4658.728889907 _step837 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated180680575638528 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed69756077 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed71420238 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current247 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9488657447936 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated189494426484736 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008KOg2- _wandbruntime4659  _runtime4659(aaM JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated71427348 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed71427086 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed180697757851648 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed185010145896114 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9239751847424 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated41344834 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 180650312 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed190187200165888 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated189512472874496 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9239664410624 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9217137489558 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed189512368398336 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9239664410624 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 110887592 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 ;4DeviceStatsMonitor.on_train_batch_end/max>_split_size-1 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 180650312 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current260 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed175793094209792 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed69762720 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 180659984 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 9489577175552 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 _step838 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated189512472874496 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 9217051686322 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated180272721027072 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 110887592 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 110897262 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated69762722 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated175793111249152 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current262 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed41344832 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 9489577175552 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 _step838 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated189512472874496 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 9217051686322 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated180272721027072 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 110887592 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 110897262 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated69762722 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated175793111249152 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current262 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed41344832 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated71427348 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed71427086 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed180697757851648 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed185010145896114 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9239751847424 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated41344834 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 180650312 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed190187200165888 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated189512472874496 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9239664410624 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9217137489558 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed189512368398336 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9239664410624 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 110887592 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 180650312 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current260 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed175793094209792 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed69762720 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 180659984:2- _wandbruntime4659  _runtime4659 U  2loss/SRqqt2/assignment_loss1.0058334770702546 /loss/FRt2/detection_loss2.5509740129693763 1loss/SRqqt1/detection_loss1.7288597926240543 /loss/FBt2/detection_loss2.0120214082611105 _runtime4659.111041489 2loss/SRqqt1/assignment_loss0.8859689938996372 4loss/FBt2/assignment_loss5.3543338053666884e-06 /loss/FBt1/detection_loss1.3283807160011654 /loss/FRt2/assignment_loss2.597485687825587 .loss/FRt1/detection_loss2.334324977806081 1loss/SRqqt2/detection_loss0.9866903843237989 epoch0 trainer/global_step10499 % _timestamp1.7857999462882087e+09 0loss/FRt1/assignment_loss2.6301287619963936 2loss/FBt1/assignment_loss0.006811124097476134 %loss/total_loss18.06748469120874 _step839Nl 4loss/FBt2/assignment_loss5.3543338053666884e-06 /loss/FBt1/detection_loss1.3283807160011654 /loss/FRt2/assignment_loss2.597485687825587 .loss/FRt1/detection_loss2.334324977806081 1loss/SRqqt2/detection_loss0.9866903843237989 epoch0 trainer/global_step10499 % _timestamp1.7857999462882087e+09 0loss/FRt1/assignment_loss2.6301287619963936 2loss/FBt1/assignment_loss0.006811124097476134 %loss/total_loss18.06748469120874 _step839 2loss/SRqqt2/assignment_loss1.0058334770702546 /loss/FRt2/detection_loss2.5509740129693763 1loss/SRqqt1/detection_loss1.7288597926240543 /loss/FBt2/detection_loss2.0120214082611105 _runtime4659.111041489 2loss/SRqqt1/assignment_loss0.8859689938996372qV|R(:  U cpu13.78282744312885 disk.drbd1584.out0 disk.drbd1289.out0 disk.drbd1510.in0 disk.sda2.out 128.09375 disk.drbd1550.in0 disk.drbd1550.out0 network.recv577437 disk.rbd1.out 18.7431640625 disk.drbd1965.in0 disk.drbd1931.in0* proc.memory.availableMB38505.74609375* proc.memory.percent17.182016372680664 disk.sda1.out0 disk.nvme0n1.in 4.64453125 network.sent 9.60242e+06 disk.rbd0.in0 disk.sda2.in0 disk.drbd1552.out0 disk.sda.in0 disk.drbd2078.out0 disk.drbd1356.out0 disk.drbd2111.in0 disk.drbd2111.out0 disk.drbd1931.out0 disk.drbd1289.in0 disk.sda1.in0# disk.nvme0n1.out191.9345703125! proc.memory.rssMB 11260.40625 disk.drbd1828.in0 disk.drbd2118.in0 disk.drbd1285.in0 disk.dm-0.out191.9345703125 disk.drbd2041.in0$ memory_percent41.24489426612854$ disk./.usageGB578.3476524353027 disk.drbd1828.out0) disk./.usagePercent32.34422834610675 disk.drbd1552.in0 proc.cpu.threads29 disk.drbd1285.out0 disk.rbd0.out0 disk.sda.out 128.09375 disk.drbd1965.out0 disk.drbd1673.out0 disk.drbd2118.out0 disk.drbd1510.out0 disk.drbd1226.in0 disk.drbd1584.in0 disk.drbd1817.in0 disk.drbd1817.out0 disk.drbd1226.out0 disk.drbd2078.in0 disk.drbd1356.in0 disk.drbd1673.in0 disk.dm-0.in 4.64453125 disk.rbd1.in0 disk.drbd2041.out0 disk.drbd1938.in0 disk.drbd1938.out02(: ܳV gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts124.228' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent41.40933333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors03j(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts125.694' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent41.898 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0GwjZ(:  U network.sent 9.647343e+06 network.recv578850 disk.drbd1285.out0 disk.drbd2078.in0 disk.sda.out 128.47265625 disk.drbd1289.out0* proc.memory.availableMB38504.99609375 disk.drbd1673.in0 disk.drbd1931.out0# disk.nvme0n1.out192.1962890625 disk.drbd1931.in0 disk.drbd1828.in0$ memory_percent41.24603867530823* proc.memory.percent17.182016372680664 cpu13.782223687267361 proc.cpu.threads29 disk.drbd1226.in0 disk.drbd1226.out0 disk.drbd2041.out0 disk.drbd2118.out0 disk.drbd2111.in0 disk.drbd1289.in0 disk.drbd1584.in0 disk.nvme0n1.in 4.64453125 disk.drbd1817.out0 disk.drbd1965.out0 disk.drbd1550.in0 disk.drbd1550.out0 disk.drbd1938.in0 disk.drbd1938.out0 disk.drbd1285.in0 disk.drbd1510.in0 disk.sda2.in0 disk.sda2.out 128.47265625 disk.drbd2041.in0 disk.drbd1828.out0! proc.memory.rssMB 11260.40625 disk.sda.in0 disk.sda1.out0 disk.drbd1356.in0 disk.drbd2111.out0 disk.drbd1817.in0* disk./.usagePercent32.344227279417545$ disk./.usageGB578.3476333618164 disk.rbd1.in0 disk.rbd0.in0 disk.drbd1510.out0 disk.drbd1584.out0 disk.rbd1.out 18.955078125 disk.rbd0.out0 disk.dm-0.out192.1962890625 disk.drbd2118.in0 disk.drbd1356.out0 disk.drbd1965.in0 disk.drbd1673.out0 disk.drbd1552.in0 disk.drbd1552.out0 disk.sda1.in0 disk.dm-0.in 4.64453125 disk.drbd2078.out0b2- _wandbruntime4680  _runtime4680)% _step840 ' lr-AdamW/pg10.00027187079523349403 ' lr-AdamW/pg20.00027187079523349403 trainer/global_step10549 % _timestamp1.7857999678207405e+09 _runtime4680.645696759t/ ' lr-AdamW/pg10.00027187079523349403 ' lr-AdamW/pg20.00027187079523349403 trainer/global_step10549 % _timestamp1.7857999678207405e+09 _runtime4680.645696759 _step840Dx2- _wandbruntime4680  _runtime4680n.r'O E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated190396760770560 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 9282780926464 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed70088227 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 111414727 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed181540906574848 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 trainer/global_step10549 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed176613362653472 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 111414727 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed181113875375104 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9282780926464 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated70088229 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current248 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed190396656301568 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed71761518 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated176613379692832 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9260062676608 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 181493298 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed70088227 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated191074823555072 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 111405071 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated41537747 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed41537745 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 _runtime4680.645789425 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated181113892414464 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated181540931478528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed181113875375104 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated30224021 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 % _timestamp1.7857999678214352e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated185873528172564 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated71761768 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current250 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed191074663782912 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed181540906574848 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 trainer/global_step10549 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed176613362653472 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 111414727 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed181113875375104 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9282780926464 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated70088229 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current248 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed190396656301568 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed71761518 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated176613379692832 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9260062676608 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 181493298 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed70088227 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated191074823555072 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 111405071 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated41537747 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed41537745 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 _runtime4680.645789425 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated181113892414464 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated181540931478528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed181113875375104 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated30224021 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 % _timestamp1.7857999678214352e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated185873528172564 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated71761768 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current250 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed191074663782912 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated190414807160320 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 111425264 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed71768293 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed176630102825792 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 trainer/global_step10549 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated190414807160320 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated70094872 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed181558113691648 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 9260940149830 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed190414702684160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed70094870 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed191092777109504 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed185891042975622 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 111415594 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated181558138595328 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 _runtime4681.025398906 _step842 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 181520136 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current270 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed41541682 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated185891145818218 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 $ _timestamp1.785799968202055e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed70094870 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 181510464 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 181520136 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current268N&M ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current268 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated190414807160320 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 111425264 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed71768293 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed176630102825792 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 trainer/global_step10549 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated190414807160320 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated70094872 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed181558113691648 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 9260940149830 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed190414702684160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed70094870 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed191092777109504 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed185891042975622 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 111415594 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated181558138595328 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 _runtime4681.025398906 _step842 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 181520136 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current270 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed41541682 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated185891145818218 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 $ _timestamp1.785799968202055e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed70094870 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 181510464 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 181520136 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0bs2- _wandbruntime4681  _runtime4681W[ /loss/FRt2/assignment_loss2.561794863541921 2loss/SRqqt1/assignment_loss0.7065559765237305 loss/FBt1/assignment_loss0 0loss/SRqqt2/detection_loss0.998361658546647 % _timestamp1.7857999682036757e+09 1loss/FBt2/assignment_loss0.21657593641429962 /loss/FRt1/detection_loss2.4493226136031905 /loss/FRt2/detection_loss2.5327989077568054 1loss/SRqqt1/detection_loss1.7172604527007391 /loss/FBt1/detection_loss0.7151230030271801 &loss/total_loss18.836428841381423 trainer/global_step10549 0loss/FRt1/assignment_loss2.5376000372987044 -loss/FBt2/detection_loss3.34089850471355 _step843 2loss/SRqqt2/assignment_loss1.0601368872546555 epoch0 _runtime 4681.0265001֚2 1loss/FBt2/assignment_loss0.21657593641429962 /loss/FRt1/detection_loss2.4493226136031905 /loss/FRt2/detection_loss2.5327989077568054 1loss/SRqqt1/detection_loss1.7172604527007391 /loss/FBt1/detection_loss0.7151230030271801 &loss/total_loss18.836428841381423 trainer/global_step10549 0loss/FRt1/assignment_loss2.5376000372987044 -loss/FBt2/detection_loss3.34089850471355 _step843 2loss/SRqqt2/assignment_loss1.0601368872546555 epoch0 _runtime 4681.0265001 /loss/FRt2/assignment_loss2.561794863541921 2loss/SRqqt1/assignment_loss0.7065559765237305 loss/FBt1/assignment_loss0 0loss/SRqqt2/detection_loss0.998361658546647 % _timestamp1.7857999682036757e+09XP(:  U disk.drbd2041.in0 disk.drbd2041.out0 disk.dm-0.in 4.64453125 disk.nvme0n1.in 4.64453125 disk.drbd2111.in0 network.sent 9.650355e+06 disk.sda.out 128.7265625 proc.cpu.threads29 disk.drbd1931.in0 disk.drbd1828.in0 disk.drbd1289.in0 disk.drbd1510.out0$ memory_percent41.24608635902405 cpu13.78141507836634 disk.drbd1356.out0 disk.drbd2111.out0* disk./.usagePercent32.344227279417545 disk.rbd0.in0 disk.sda2.in0 disk.rbd1.in0 disk.drbd1673.in0 disk.drbd1552.out0 disk.drbd1285.in0 disk.drbd1931.out0 disk.dm-0.out192.3681640625 disk.drbd1356.in0 disk.drbd1965.out0 disk.drbd1584.in0# disk.nvme0n1.out192.3681640625* proc.memory.percent17.182016372680664 disk.drbd1817.out0 disk.rbd0.out0 disk.sda.in0 disk.sda1.in0 disk.drbd1584.out0! proc.memory.rssMB 11260.40625 disk.drbd1552.in0 disk.drbd1828.out0 dis*@k.drbd2078.out0 disk.drbd1965.in0 disk.drbd1289.out0 network.recv579756 disk.drbd1673.out0 disk.sda1.out0 disk.drbd1226.out0 disk.drbd2078.in0 disk.drbd2118.out0 disk.drbd1817.in0 disk.drbd1285.out0 disk.drbd1510.in0$ disk./.usageGB578.3476333618164 disk.drbd1550.in0 disk.drbd1938.in0 disk.drbd1226.in0 disk.drbd2118.in0 disk.sda2.out 128.7265625 disk.rbd1.out 18.955078125 disk.drbd1550.out0 disk.drbd1938.out0* proc.memory.availableMB38504.96484375q顐(: U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts119.121' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent39.707 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0{2- _wandbruntime4702  _runtime4702Z  _step844 & lr-AdamW/pg10.0002731594045577593 & lr-AdamW/pg20.0002731594045577593 trainer/global_step10599 $ _timestamp1.785799989822193e+09 _runtime4702.6466979514b & lr-AdamW/pg20.0002731594045577593 trainer/global_step10599 $ _timestamp1.785799989822193e+09 _runtime4702.646697951 _step844 & lr-AdamW/pg10.0002731594045577593S2- _wandbruntime4702  _runtime4702':'O `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated177450388308832 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed41734595 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated30368185 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed181972213525504 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current251 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated191299095056384 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 _step845 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed30367936 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 9578945717760 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated182401287318528 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 111942729 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 182353450 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed181972213525504 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 182363108 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current249 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 9304036943240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 trainer/global_step10599 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 9326864491520 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 _runtime4702.646788799 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed191980208132608 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 9326864491520 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9326777061888 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated191980372099072 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated181972230564864 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9579084780544 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated186754425252072 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed177450371269472 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated70420379 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated191299095056384 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9303951140116 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated70420379 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed191298990587392 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated181972230564864 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated41734597 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated72102782 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 111942729 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 9326777061888 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed70420377 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed70420377 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed186754322409588 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed181972213525504 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current251 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated191299095056384 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 _step845 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed30367936 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 9578945717760 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated182401287318528 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 111942729 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 182353450 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed181972213525504 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 182363108 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current249 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 9304036943240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 trainer/global_step10599 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 9326864491520 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 _runtime4702.646788799 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed191980208132608 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 9326864491520 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9326777061888 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated191980372099072 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated181972230564864 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9579084780544 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated186754425252072 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed177450371269472 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated70420379 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated191299095056384 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9303951140116 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated70420379 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed191298990587392 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated181972230564864 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated41734597 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated72102782 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 111942729 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 9326777061888 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed70420377 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed70420377 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed186754322409588 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9327744118272 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated182418494435328 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9327744118272 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated191998478897664 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current252 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed72109260 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 _step846 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9327656681472 $ _timestamp1.785799990210908e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated70427022 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated181989397327872 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9327656681472 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated191317141446144 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated186772042897726 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated70427022 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed70427020 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 182380288 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 182370616 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated30370978 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed181989380288512 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.allg#.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9327744118272 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated182418494435328 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9327744118272 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated191998478897664 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current252 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed72109260 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 _step846 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9327656681472 $ _timestamp1.785799990210908e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated70427022 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated181989397327872 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9327656681472 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated191317141446144 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated186772042897726 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated70427022 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed70427020 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 182380288 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024Qmg2- _wandbruntime4703  _runtime4703i?o  1loss/SRqqt1/assignment_loss1.066350520460629 2loss/SRqqt2/assignment_loss0.9955789629102816 3loss/FBt1/assignment_loss0.0008085576241569039 1loss/SRqqt2/detection_loss0.9676529330273087 _step847 0loss/FRt1/assignment_loss2.4652941241321793 4loss/FBt2/assignment_loss1.2728978611217404e-05 1loss/SRqqt1/detection_loss1.6425869200086476 /loss/FBt2/detection_loss1.0080540896409989 %loss/total_loss16.46887324105623 _runtime4703.035443136 /loss/FRt1/detection_loss2.5997837464134377 .loss/FRt2/detection_loss2.410407941424992 /loss/FBt1/detection_loss0.9214639002316152 trainer/global_step10599 % _timestamp1.7857999902125971e+09 /loss/FRt2/assignment_loss2.390878816203373 epoch0>S( trainer/global_step10599 % _timestamp1.7857999902125971e+09 /loss/FRt2/assignment_loss2.390878816203373 epoch0 1loss/SRqqt1/assignment_loss1.066350520460629 2loss/SRqqt2/assignment_loss0.9955789629102816 3loss/FBt1/assignment_loss0.0008085576241569039 1loss/SRqqt2/detection_loss0.9676529330273087 _step847 0loss/FRt1/assignment_loss2.4652941241321793 4loss/FBt2/assignment_loss1.2728978611217404e-05 1loss/SRqqt1/detection_loss1.6425869200086476 /loss/FBt2/detection_loss1.0080540896409989 %loss/total_loss16.46887324105623 _runtime4703.035443136 /loss/FRt1/detection_loss2.5997837464134377 .loss/FRt2/detection_loss2.410407941424992 /loss/FBt1/detection_loss0.9214639002316152cd(: ЯU gpu.0.gpu38.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts126.849' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent42.283 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0%w\(:  U$ disk./.usageGB578.3476333618164 disk.sda.in0 disk.sda.out 129.05078125 disk.drbd1938.in0 disk.drbd1356.in0 disk.drbd1356.out0 disk.drbd1965.out0 disk.drbd1931.in0 disk.drbd1552.out0 disk.drbd1289.in0 disk.drbd1965.in0 disk.drbd1285.out0 disk.drbd1931.out0 disk.drbd1828.out0 disk.drbd1938.out0 disk.rbd0.out0 disk.sda1.out0 disk.drbd2111.in0 disk.drbd1673.out0! disk.dm-0.out192.69091796875$ disk.nvme0n1.out192.69091796875 disk.rbd1.in0 disk.rbd1.out 19.017578125 cpu13.780738829094492 proc.cpu.threads29 disk.drbd1828.in0 disk.sda1.in0 disk.drbd1584.in0 disk.drbd2118.in0 disk.drbd2111.out0 network.sent 9.695267e+06 disk.drbd1550.in0 disk.drbd1226.out0 disk.drbd2118.out0 disk.drbd1817.in0 disk.drbd2041.in0 disk.drbd1510.out0 disk.sda2.in0 disk.drbd1673.in0 disk.drbd1289.out0 disk.drbd1226.in0 disk.drbd2078.out0 disk.drbd1584.out0! proc.memory.rssMB 11260.40625* disk./.usagePercent32.344227279417545* proc.memory.percent17.182016372680664 disk.rbd0.in0 disk.dm-0.in 4.64453125 disk.drbd2078.in0 disk.drbd1817.out0 network.recv581201 disk.drbd1550.out0 disk.drbd1285.in0 disk.drbd1510.in0 disk.nvme0n1.in 4.64453125) proc.memory.availableMB 38504.9296875 disk.drbd2041.out0 disk.drbd1552.in0 disk.sda2.out 129.05078125% memory_percent41.246140003204346HCU(: ʕU gpu.0.gpu24.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts120.378' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent40.126 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0R(:  U disk.drbd1673.out0 disk.drbd1552.out0 disk.drbd2111.in0 disk.drbd1550.in0 disk.sda.in0 disk.drbd2078.in0 disk.drbd1584.out0 disk.rbd1.in0 disk.drbd1289.out0 disk.rbd0.out0 disk.drbd1965.in0 disk.drbd1965.out0 disk.drbd1673.in0 disk.drbd1931.in0 disk.drbd1552.in0 disk.drbd1226.out0 disk.drbd1356.in0 disk.drbd2078.out0 disk.drbd2041.out0 disk.drbd1828.out0 disk.drbd1938.in0 cpu13.77991344643418 disk.sda2.in0 disk.drbd1510.out0* proc.memory.percent17.182016372680664) disk./.usagePercent32.34422834610675 disk.drbd2041.in0 disk.drbd1510.in0 network.recv582754 disk.sda2.out 129.359375 disk.drbd1285.in0! disk.dm-0.out192.82373046875$ disk.nvme0n1.out192.82373046875 disk.drbd1226.in0 disk.drbd1356.out0 disk.drbd1584.in0 disk.drbd1817.in0 disk.drbd1938.out0 proc.cpu.threads29 disk.drbd1817.out0 disk.rbd1.out 19.017578125 disk.drbd1550.out0 disk.drbd1289.in0 disk.sda1.out0% proc.memory.availableMB 38505.375 network.sent 9.740249e+06 disk.nvme0n1.in 4.64453125 disk.drbd1828.in0 disk.sda.out 129.359375 disk.dm-0.in 4.64453125! proc.memory.rssMB 11260.40625 disk.drbd2118.out0% memory_percent41.245460510253906$ disk./.usageGB578.3476524353027 disk.drbd2111.out0 disk.rbd0.in0 disk.sda1.in0 disk.drbd1285.out0 disk.drbd2118.in0 disk.drbd1931.out0j2- _wandbruntime4725  _runtime4725 ]t ' lr-AdamW/pg20.00027444801388202465 trainer/global_step10649 $ _timestamp1.785800012354209e+09 _runtime4725.177211844 _step848 ' lr-AdamW/pg10.00027444801388202465{y ' lr-AdamW/pg10.00027444801388202465 ' lr-AdamW/pg20.00027444801388202465 trainer/global_step10649 $ _timestamp1.785800012354209e+09 _runtime4725.177211844 _step848J2- _wandbruntime4725  _runtime4725.'O _step849 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9370773197312 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9347839603624 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed70752527 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated192201429342208 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 183223260 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 % _timestamp1.7858000123552158e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated70752529 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current257 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed192885737512960 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed182830551675904 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed192201324873216 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current255 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed183261618254848 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 112461075 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated192201429342208 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated182830568715264 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated182830568715264 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 112470731 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 112470731 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed182830551675904 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed72441728 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 trainer/global_step10649 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated30510538 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated70752529 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated183261643158528 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated178287396924832 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated41931447 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 112461075 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed192201324873216 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 9370860626944 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated72441985 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9624252029440 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 9624119258112 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed70752527 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 _runtime4725.178434592 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed178287379885472 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 9370860626944 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated192885895187968 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated187635322331580 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0|O JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 9370860626944 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated72441985 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9624252029440 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 9624119258112 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed70752527 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 _runtime4725.178434592 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed178287379885472 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 9370860626944 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated192885895187968 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated187635322331580 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 _step849 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9370773197312 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9347839603624 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed70752527 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated192201429342208 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 183223260 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 % _timestamp1.7858000123552158e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated70752529 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current257 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed192885737512960 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed182830551675904 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed192201324873216 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current255 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed183261618254848 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 112461075 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated192201429342208 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated182830568715264 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated182830568715264 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 112470731 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 112470731 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed182830551675904 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed72441728 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 trainer/global_step10649 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 G?DeviceStatsMonitor.on_train_batch_start/inacta.Z ive_split.all.peak548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated30510538 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated70752529 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated183261643158528 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated178287396924832 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated41931447 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 112461075 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed192201324873216 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9371740253696 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9348802880082 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 trainer/global_step10649 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 % _timestamp1.7858000127290654e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated70759172 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed182847718438912 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed192219371255808 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated192219475731968 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated70759172 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed41935382 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed187652837134638 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 112481268 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated192904004018176 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed178304120057792 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 9348717076846 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 183230768 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed72448517 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 112471598 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 _step850 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed182847718438912 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9371652816896 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 183240440 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed192903842156032 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 183240440 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 9625016784384 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 183230768 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed30513135 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated178304137097152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9371740253696 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9348802880082 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 trainer/global_step10649 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 % _timestamp1.7858000127290654e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated70759172 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed182847718438912 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed192219371255808 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated192219475731968 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated70759172 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed41935382 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed187652837134638 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 112481268 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated192904004018176 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed178304120057792 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 9348717076846 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 183230768 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed72448517 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 112471598 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 _step850 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed182847718438912 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9371652816896 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 183240440 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed192903842156032 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 183240440 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 9625016784384 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 183230768 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed30513135 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated178304137097152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 FDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed193103659159040 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed183688889826304 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 112989077 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current250 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 % _timestamp1.7858000345241077e+09 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current252 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated179124405540832 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 112998733 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated183688906865664 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated42128297 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9669440107008 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated193791439105536 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated193103763628032 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 112989077 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 9391813870256 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed188516116568604 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 9414856762368 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 9414769332736 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 184083412 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed71084677 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed193791279333376 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated183688906865664 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated30653947 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9391728067132 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated184121998998528 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 112998733 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 NFDeviceStatsMonitor.on_trainxM_batch_start/inactive_split.small_pool.peak538 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated71084679 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed183688889826304 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _runtime 4747.3477316 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed71084677 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed184121974094848 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 184073754 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed179124388501472:, 'O UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed188516116568604 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 9414856762368 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 9414769332736 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 184083412 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed71084677 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed193791279333376 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated183688906865664 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated30653947 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9391728067132 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated184121998998528 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 112998733 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated71084679 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed183688889826304 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _runtime 4747.3477316 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed71084677 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed184121974094848 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 184073754 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed179124388501472 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed193103659159040 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed183688889826304 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 112989077 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current250 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 % _timestamp1.7858000345241077e+09 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current252 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated179124405540832 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 112998733 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated183688906865664 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated42128297 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9669440107008 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated193791439105536 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated193103763628032 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 112989077 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 9391813870256 k2- _wandbruntime4747  _runtime4747*f&M OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated179141145713152 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated183706073628672 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed42132232 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 9670209646080 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated193121810017792 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 184100592 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 184090920 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed179141128673792 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated42132234 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 trainer/global_step10699 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 9392605540354 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 % _timestamp1.7858000348964446e+09 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9415648952320 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed71091320 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 112999600 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed72788765 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated183706073628672 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed183706056589312 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 113009270 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 _step854 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated184139206115328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed71091320 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9392691343590 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 _runtime 4747.7197898:8`M S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed193121705541632 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9415648952320 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated71091322 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated188533837056742 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 112999600 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9415736389120 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated183706073628672 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed183706056589312 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 113009270 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 _step854 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated184139206115328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed71091320 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9392691343590 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 _runtime 4747.7197898 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated179141145713152 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated183706073628672 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed42132232 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 9670209646080 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated193121810017792 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 184100592 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 184090920 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed179141128673792 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated42132234 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 trainer/global_step10699 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 9392605540354 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 % _timestamp1.7858000348964446e+09 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9415648952320 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed71091320 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 112999600 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed72788765 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 F;S(:  U disk.sda2.out 130.296875 disk.drbd1828.out0 disk.sda1.in0 disk.sda1.out0 disk.drbd1356.in0 disk.drbd1938.out0 disk.drbd2118.out0 disk.drbd1817.in0 disk.rbd1.out 19.2060546875 disk.drbd1289.out0 disk.drbd1673.in0 disk.drbd1828.in0 disk.sda.in0 disk.dm-0.in 4.64453125 disk.drbd2078.in0 disk.drbd1817.out0 disk.drbd2041.in0 disk.dm-0.out 193.931640625 disk.drbd1965.out0 disk.drbd1226.out0 disk.nvme0n1.in 4.64453125 disk.drbd1673.out0 disk.rbd0.in0 disk.rbd0.out0 disk.drbd1584.in0! proc.memory.rssMB 11260.40625 disk.sda2.in0 disk.drbd1965.in0 disk.drbd1552.out0 disk.drbd2078.out0( disk./.usagePercent32.3442061589713" disk.nvme0n1.out 193.931640625 disk.drbd2041.out0 disk.sda.out 130.296875 disk.drbd1285.in0 disk.drbd1550.in0 disk.drbd1289.in0 disk.drbd1510.in0 network.sent 9.833284e+06$ disk./.usageGB578.3472557067871 disk.drbd2111.out0$ memory_percent41.24615788459778 cpu13.782272494038232 disk.drbd1550.out0 disk.drbd1510.out0 disk.drbd1226.in0 disk.drbd1584.out0 disk.drbd2118.in0* proc.memory.percent17.182016372680664 disk.drbd1356.out0 disk.rbd1.in0 disk.drbd1931.in0 disk.drbd1552.in0 disk.drbd1938.in0 disk.drbd1285.out0* proc.memory.availableMB38504.91796875 network.recv586836 disk.drbd2111.in0 disk.drbd1931.out0 proc.cpu.threads29ۉ(: U gpu.0.gpu38.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts99.484' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent33.16133333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0rp2- _wandbruntime4770  _runtime4770  _runtime4770.024497143 _step856 & lr-AdamW/pg10.0002770252325305553 & lr-AdamW/pg20.0002770252325305553 trainer/global_step10749 % _timestamp1.7858000572015254e+09йi\ _step856 & lr-AdamW/pg10.0002770252325305553 & lr-AdamW/pg20.0002770252325305553 trainer/global_step10749 % _timestamp1.7858000572015254e+09 _runtime4770.0244971436G1|2- _wandbruntime4770  _runtime4770#1/'O JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed30797867 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 9435702333764 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9435616530640 _step857 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 113517079 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 9714476896768 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9714618056704 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed194005993444864 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed194005993444864 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated30798130 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed184982329934848 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 184943564 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed179961397117472 % _timestamp1.7858000572025156e+09 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated184982354838528 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9458765468160 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated179961414156832 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated194006097913856Ǘ`'O UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 113526735 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current263 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed42325145 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 166063616 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed30797867 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 9435702333764 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9435616530640 _step857 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 113517079 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 9714476896768 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9714618056704 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed194005993444864 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed194005993444864 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated30798130 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed184982329934848 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 184943564 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed179961397117472 % _timestamp1.7858000572025156e+09 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated184982354838528 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9458765468160 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated179961414156832 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated194006097913856 JDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 184960744 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated194024144303616 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated179978154329152 _step858 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated184564411779072 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 % _timestamp1.7858000575806584e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated73130061 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated30800977 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 184960744 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed30800724 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated184999561955328 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 \HDeviceStatsMonit+7Mor.on_train_batch_end/inactive_split_bytes.all.allocated194715075059200 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed184999537051648 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 9715378242560 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 184951072 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated194024144303616 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed73129806 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated42329084 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed184564394739712 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 9715513103872 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 113537272 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9459645087744 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current253 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9436579807098 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated71423472 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 9436494003862 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9459732524544 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated71423472 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed179978137289792 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed71423470 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9459732524544 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 113527602 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed71423470 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed184564394739712 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed42329082 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated189414734136250 trainer/global_step10749K.&M [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed184564394739712 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed42329082 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated189414734136250 trainer/global_step10749 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9459645087744 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 184960744 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated194024144303616 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated179978154329152 _step858 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated184564411779072 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 % _timestamp1.7858000575806584e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated73130061 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated30800977 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 184960744 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed30800724 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated184999561955328 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated194715075059200 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed184999537051648 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 9715378242560 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 184951072 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated194024144303616 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed73129806 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated42329084 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed184564394739712 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 9715513103872 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 113537272 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9459645087744 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current253 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9436579807098 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated71423472 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 9436494003862 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9459732524544 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated71423472 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed179978137289792 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed71423470 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9459732524544 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 113527602 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed71423470ީ2- _wandbruntime4770  _runtime4770K  2loss/SRqqt2/assignment_loss0.9460636059634555 1loss/SRqqt2/detection_loss0.9740763282949539 %loss/total_loss17.93025683292447 % _timestamp1.7858000575823245e+09 /loss/FBt1/detection_loss1.0922353583764994 /loss/FBt2/detection_loss2.1389540206227036 _step859 0loss/FRt1/assignment_loss2.7578159058705354 3loss/FBt1/assignment_loss0.0002505800578481128 /loss/FRt1/detection_loss2.5716541611995454 epoch0 trainer/global_step10749 /loss/FRt2/assignment_loss2.111999646574259 1loss/SRqqt1/assignment_loss1.091145936618833 3loss/FBt2/assignment_loss3.561177633236589e-05 .loss/FRt2/detection_loss2.430085505358875 1loss/SRqqt1/detection_loss1.8159401722106279 _runtime4770.405155882폫 3loss/FBt2/assignment_loss3.561177633236589e-05 .loss/FRt2/detection_loss2.430085505358875 1loss/SRqqt1/detection_loss1.8159401722106279 _runtime4770.405155882 2loss/SRqqt2/assignment_loss0.9460636059634555 1loss/SRqqt2/detection_loss0.9740763282949539 %loss/total_loss17.93025683292447 % _timestamp1.7858000575823245e+09 /loss/FBt1/detection_loss1.0922353583764994 /loss/FBt2/detection_loss2.1389540206227036 _step859 0loss/FRt1/assignment_loss2.7578159058705354 3loss/FBt1/assignment_loss0.0002505800578481128 /loss/FRt1/detection_loss2.5716541611995454 epoch0 trainer/global_step10749 /loss/FRt2/assignment_loss2.111999646574259 1loss/SRqqt1/assignment_loss1.0911459366188330s (: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts119.81' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent39.93666666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0`cX(:  U disk.sda2.out 130.73828125 disk.rbd0.out0! proc.memory.rssMB 11260.40625 disk.drbd1510.in0 disk.drbd1584.in0 disk.drbd1965.in0 disk.sda1.in0 disk.drbd1584.out0 disk.sda.in0 disk.dm-0.in 4.64453125 disk.drbd2078.in0 disk.drbd1817.in0 disk.drbd1828.out0$ disk./.usageGB578.3474998474121 disk.drbd1285.in0 disk.drbd1510.out0 disk.nvme0n1.in 4.64453125 disk.drbd1673.out0 disk.drbd1938.out0 disk.drbd2078.out0$ memory_percent41.24659299850464 disk.drbd2041.out0 disk.drbd1552.in0 disk.drbd1938.in0 disk.rbd0.in0 disk.sda.out 130.73828125 proc.cpu.threads29 disk.drbd1226.out0 disk.drbd1356.out0 disk.drbd1931.out0$ disk.nvme0n1.out194.54052734375 disk.drbd1226.in0 disk.drbd1817.out0 disk.rbd1.in0 disk.drbd1931.in0 disk.drbd1550.in0! disk.dm-0.out194.54052734375 disk.drbd2111.out0* proc.memory.percent17.182016372680664 network.recv587656) disk./.usagePercent32.34421981259312 disk.drbd1673.in0 disk.drbd1828.in0 disk.drbd1965.out0 disk.drbd2041.in0 disk.drbd1289.out0) proc.memory.availableMB 38504.6328125 disk.drbd1356.in0 network.sent 9.8363e+06 disk.drbd2118.in0 disk.drbd2118.out0 disk.rbd1.out 19.2060546875 disk.drbd2111.in0 disk.drbd1552.out0 disk.drbd1289.in0 cpu13.78315002816793 disk.drbd1285.out0 disk.drbd1550.out0 disk.sda1.out0 disk.sda2.in0k2- _wandbruntime4792  _runtime4792B7 trainer/global_step10799 % _timestamp1.7858000796584344e+09 _runtime4792.481393885 _step860 ' lr-AdamW/pg10.00027831384185482053 ' lr-AdamW/pg20.00027831384185482053 ' lr-AdamW/pg20.00027831384185482053 trainer/global_step10799 % _timestamp1.7858000796584344e+09 _runtime4792.481393885 _step860 ' lr-AdamW/pg10.00027831384185482053^h!2- _wandbruntime4792  _runtime4792.'O SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated42521997 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed185842685774848 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed180798405733472 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _runtime4792.482703576 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed71748977 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 9502849033216 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed194908327730688 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 9502849033216 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated71748979 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed30941528 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated185405583166464 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 9759646458368 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed185405566127104 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 trainer/global_step10799 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 114054737 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated195602487811072 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed190277910727620 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 9479590797272 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated185405583166464 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current248 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated185842710678528 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated194908432199680 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated73463771 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 185794058 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated71748979 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 185803716 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed73463523 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current246 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9502761603584 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed42521995 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9759777132544wq O N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9759777132544 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated42521997 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed185842685774848 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed180798405733472 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _runtime4792.482703576 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed71748977 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 9502849033216 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed194908327730688 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 9502849033216 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated71748979 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed30941528 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated185405583166464 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 9759646458368 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed185405566127104 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 trainer/global_step10799 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 114054737 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated195602487811072 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed190277910727620 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 9479590797272 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated185405583166464 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current248 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated185842710678528 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated194908432199680 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated73463771 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 185794058 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated71748979 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 185803716 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed73463523 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current246 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9502761603584 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed42521995 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated180815162945152 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated185422749929472 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated73470565 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9503641223168 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated71755622 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current255 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed71755620 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 _runtime4792.849415539 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 % _timestamp1.7858000800260735e+09 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed185422732890112 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 185811224 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 114065274 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 185820896 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated42525934 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed30944376 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed180815145905792 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 185811224 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated194926478589440 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed73470308 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 9480382467370 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated71755622 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9503728659968 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 9760543566848 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 114065274 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 trainer/global_step10799 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 185820896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9503728659968 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed194926374113280 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed71755620 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed195620436458496 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed185422732890112 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed185859892891648 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 9760680525312 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed190295528373162 _step862 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current257 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251:n&M OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 114065274 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 trainer/global_step10799 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 185820896 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9503728659968 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed194926374113280 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed71755620 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed195620436458496 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed185422732890112 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed185859892891648 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 9760680525312 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed190295528373162 _step862 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current257 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 114055604 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 114055604 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated180815162945152 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated185422749929472 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated73470565 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9503641223168 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated71755622 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current255 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed71755620 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 _runtime4792.849415539 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 % _timestamp1.7858000800260735e+09 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed185422732890112 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 185811224 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 114065274 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 185820896 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated42525934 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed30944376 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed180815145905792 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 185811224 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated194926478589440 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed73470308 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 9480382467370 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated71755622 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9503728659968 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 9760543566848 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856Z2- _wandbruntime4792  _runtime4792g  1loss/SRqqt2/detection_loss0.9827014435395286 1loss/SRqqt1/detection_loss1.9547204550769595 /loss/FBt1/detection_loss1.4492591660004108 trainer/global_step10799 % _timestamp1.7858000800280368e+09 2loss/SRqqt2/assignment_loss0.8759410517238133 3loss/FBt2/assignment_loss0.0001572599700716231 /loss/FRt1/detection_loss2.5032668032919068 /loss/FBt2/detection_loss1.0937554189669234 &loss/total_loss17.050364046725157 _runtime4792.850879121 _step863 /loss/FRt2/assignment_loss2.471069656230591 1loss/FBt1/assignment_loss0.01024942497170637 .loss/FRt2/detection_loss2.291334812334933 epoch0 0loss/FRt1/assignment_loss2.5799551914973433 2loss/SRqqt1/assignment_loss0.83795336312096984C epoch0 0loss/FRt1/assignment_loss2.5799551914973433 2loss/SRqqt1/assignment_loss0.8379533631209698 1loss/SRqqt2/detection_loss0.9827014435395286 1loss/SRqqt1/detection_loss1.9547204550769595 /loss/FBt1/detection_loss1.4492591660004108 trainer/global_step10799 % _timestamp1.7858000800280368e+09 2loss/SRqqt2/assignment_loss0.8759410517238133 3loss/FBt2/assignment_loss0.0001572599700716231 /loss/FRt1/detection_loss2.5032668032919068 /loss/FBt2/detection_loss1.0937554189669234 &loss/total_loss17.050364046725157 _runtime4792.850879121 _step863 /loss/FRt2/assignment_loss2.471069656230591 1loss/FBt1/assignment_loss0.01024942497170637 .loss/FRt2/detection_loss2.291334812334933#1q(: 鵶U gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts96.052' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent32.01733333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0 X(:  U network.recv589503 disk.drbd1828.in0 disk.sda2.out 131.40234375 disk.rbd1.in0 disk.drbd1673.out0 cpu13.782404913254593 disk.rbd0.in0 disk.drbd1510.out0 disk.drbd1673.in0 disk.drbd1931.in0 proc.cpu.threads29 disk.drbd2041.in0 disk.drbd1550.out0$ disk.nvme0n1.out203.90576171875 disk.drbd1356.out0 disk.drbd1285.out0 disk.drbd1938.in0 disk.drbd1828.out0! disk.dm-0.out203.90576171875 disk.drbd1226.out0 disk.drbd2118.in0 disk.drbd1817.in0 disk.sda.out 131.40234375 disk.nvme0n1.in 4.64453125 disk.drbd1356.in0! proc.memory.rssMB 11260.40625* disk./.usagePercent32.344218745903916 disk.drbd1289.in0 disk.drbd1965.in0 disk.drbd1289.out0 disk.drbd1584.in0 disk.drbd1965.out0 disk.drbd1931.out0 disk.drbd1510.in0 disk.drbd1226.in0 disk.rbd1.out 19.33203125$ memory_percent41.24703407287598 network.sent 9.881287e+06 disk.rbd0.out0 disk.drbd1285.in0 disk.drbd2078.in0 disk.drbd2118.out0 disk.drbd2111.out0' proc.memory.availableMB 38504.34375 disk.drbd1552.in0 disk.drbd1550.in0 disk.sda1.out0 disk.dm-0.in 4.64453125 disk.sda2.in0 disk.drbd1817.out0* proc.memory.percent17.182016372680664 disk.drbd2111.in0$ disk./.usageGB578.3474807739258 disk.drbd1552.out0 disk.drbd1938.out0 disk.sda.in0 disk.sda1.in0 disk.drbd2078.out0 disk.drbd1584.out0 disk.drbd2041.out0; 2- _wandbruntime4814  _runtime4814`9q ' lr-AdamW/pg10.00027960245117908584 ' lr-AdamW/pg20.00027960245117908584 trainer/global_step10849 % _timestamp1.7858001020460498e+09 _runtime4814.869038968 _step8644  % _timestamp1.7858001020460498e+09 _runtime4814.869038968 _step864 ' lr-AdamW/pg10.00027960245117908584 ' lr-AdamW/pg20.00027960245117908584 trainer/global_step10849+2- _wandbruntime4814  _runtime4814_O N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 trainer/global_step10849 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed195810662016512 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 186654210 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 @8DeviceStaGb,%tsMonitor.on_train_batch_start/num_device_alloc251 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated181635431388832 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed72081127 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 9546845168640 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed186263904277504 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated72081129 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated195810766485504 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated42718847 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated186263921316864 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated72081129 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed73803948 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed196507829557760 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated186703066518528 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated73804214 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated195810766485504 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated186263921316864 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 % _timestamp1.7858001020470212e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated191158910649612 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 114573083 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed72081127 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current266 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed42718845 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed31085103 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed195810662016512 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed186263904277504 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 114573083 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 9523479260780 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 9546757739008 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 _runtime4814.870902131 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9804922811392 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 9804787942912 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9523393457656 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 114582739 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed191158807807128 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed181635414349472 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 _step865_4'O ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated196507989329920 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed186263904277504 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 114573083 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 9523479260780 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 9546757739008 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 _runtime4814.870902131 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9804922811392 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 9804787942912 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9523393457656 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 114582739 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed191158807807128 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed181635414349472 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 _step865 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 trainer/global_step10849 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed195810662016512 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 186654210 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated181635431388832 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed72081127 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 9546845168640 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed186263904277504 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated72081129 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated195810766485504 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated42718847 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated186263921316864 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated72081129 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed73803948 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed196507829557760 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated186703066518528 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated73804214 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated195810766485504 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated186263921316864 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 % _timestamp1.7858001020470212e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated191158910649612 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 114573083 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed72081127 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current266 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed42718845 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed31085103 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed195810662016512 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9547637358592 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 186671376 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 186681048 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9547637358592 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 _step866 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated195828812875264 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed186281071040512 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 114593276 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated196526095292928 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed186281071040512 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9547724795392 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated195828812875264 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 114593276 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated72087772 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 % _timestamp1.7858001024256773e+09 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated186281088079872 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated42722784 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated31088203 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed186720248731648 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed73810732 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9547724795392 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed42722782 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed196525939722240 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 186681048 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 9805821657600 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated73810987 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9524356734114 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0"M TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated186720273635328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 114583606 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 186681048 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 9805821657600 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated73810987 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9524356734114 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9547637358592 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 186671376 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 186681048 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9547637358592 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 _step866 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated195828812875264 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed186281071040512 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 114593276 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated196526095292928 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed186281071040512 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9547724795392 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated195828812875264 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 114593276 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated72087772 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 % _timestamp1.7858001024256773e+09 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated186281088079872 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated42722784 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated31088203 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed186720248731648 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed73810732 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9547724795392 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed42722782 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed196525939722240 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 PDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 115110741 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated31228658 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed72413277 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 115101085 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed192039704886636 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 _runtime4837.346727665 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 9850012029952 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated182472440004832 _step869 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated196713100771328 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated74144355 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9850138509824 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated187122259467264 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed187122242427904 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated72413279 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed74144099 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated192039807729120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 9590841304064 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current254 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed187563397454848 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9590753874432 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated187563422358528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9567281921164 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 115101085 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 $ _timestamp1.785800124523378e+09 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed187122242427904 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 9590841304064 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 187514362 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed196712996302336 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed196712996302336 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed72413277 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed42915695 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 trainer/global_step10899 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated74144355 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9850138509824 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated187122259467264 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed187122242427904 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated72413279 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed74144099 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated192039807729120 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 9590841304064 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current254 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed187563397454848 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9590753874432 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated187563422358528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9567281921164 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 115101085 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 $ _timestamp1.785800124523378e+09 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed187122242427904 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 9590841304064 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 187514362 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed196712996302336 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed196712996302336 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed72413277 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed42915695 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 trainer/global_step10899 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 115110741 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated312286582- _wandbruntime4837  _runtime4837k&M E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _runtime4837.734341055 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated197431674957824 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed197431521484288 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated196731147161088 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 115111608 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated72419922 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated196731147161088 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed31231238 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9591720930816 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9591633494016 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed74150870 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current235 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated182489180177152 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9568245197622 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated187139426230272 _step870 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed187580604571648 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated187139426230272 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9591720930816 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated74151107 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated187580629475328 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 187541200 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current237 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated72419922 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 187531528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 187531528 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed192057322532178 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed72419920 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated31231473 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9591633494016 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 trainer/global_step10899 % _timestamp1.7858001249109638e+09 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed74150870 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current235 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated182489180177152 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9568245197622 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated187139426230272 _step870 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed187580604571648 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated187139426230272 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9591720930816 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated74151107 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated187580629475328 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 187541200 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current237 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated72419922 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 187531528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 187531528 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed192057322532178 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed72419920 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated31231473 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9591633494016 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 trainer/global_step10899 % _timestamp1.7858001249109638e+09 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated197431674957824 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed197431521484288 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated196731147161088 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 115111608 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated72419922 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated196731147161088 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed31231238 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936R^2- _wandbruntime4837  _runtime4837m[c /loss/FBt2/detection_loss1.9407590693528098 %loss/total_loss17.80948453219655 0loss/FRt2/assignment_loss2.1009782552719116 2loss/SRqqt2/assignment_loss0.8620486202533357 2loss/FBt2/assignment_loss0.004214550501512601 .loss/FRt2/detection_loss2.880175357684493 _runtime4837.735740529 _step871 4loss/FBt1/assignment_loss0.00014170628632855417 .loss/FRt1/detection_loss2.679511093431049 trainer/global_step10899 % _timestamp1.7858001249128942e+09 2loss/SRqqt1/assignment_loss1.0840759254608523 0loss/SRqqt1/detection_loss1.626906456024039 1loss/SRqqt2/detection_loss0.9609505043431584 /loss/FBt1/detection_loss1.1795275382125208 /loss/FRt1/assignment_loss2.490195455374541 epoch0   1loss/SRqqt2/detection_loss0.9609505043431584 /loss/FBt1/detection_loss1.1795275382125208 /loss/FRt1/assignment_loss2.490195455374541 epoch0 /loss/FBt2/detection_loss1.9407590693528098 %loss/total_loss17.80948453219655 0loss/FRt2/assignment_loss2.1009782552719116 2loss/SRqqt2/assignment_loss0.8620486202533357 2loss/FBt2/assignment_loss0.004214550501512601 .loss/FRt2/detection_loss2.880175357684493 _runtime4837.735740529 _step871 4loss/FBt1/assignment_loss0.00014170628632855417 .loss/FRt1/detection_loss2.679511093431049 trainer/global_step10899 % _timestamp1.7858001249128942e+09 2loss/SRqqt1/assignment_loss1.0840759254608523 0loss/SRqqt1/detection_loss1.6269064560240395*ʛ(: U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts124.169' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent41.38966666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0@G(:  V disk.sda2.out 132.546875 disk.drbd1965.out0 disk.drbd1938.out0* proc.memory.percent17.182016372680664 disk.drbd1552.out0 network.sent 9.995489e+06* disk./.usagePercent32.344230266147314 disk.sda1.in0 disk.drbd1673.in0 disk.drbd1673.out0 disk.drbd2041.out0 disk.sda.in0 disk.rbd1.out 19.57421875 disk.rbd0.in0 disk.sda1.out0 disk.nvme0n1.in 4.64453125 disk.drbd1550.out0 disk.dm-0.out 205.640625 disk.drbd2118.in0 disk.nvme0n1.out 205.640625 disk.drbd1356.in0 disk.drbd1356.out0 disk.sda2.in0 proc.cpu.threads29 disk.rbd0.out0 disk.drbd1828.out0 disk.drbd2078.out0 disk.drbd1817.in0 disk.drbd1931.in0 disk.drbd1931.out0 disk.drbd1828.in0 disk.drbd1285.out0 disk.drbd1510.out0 disk.drbd1285.in0 disk.drbd1817.out0! proc.memory.rssMB 11260.40625$ disk./.usageGB578.3476867675781 disk.drbd1226.in0 disk.drbd2078.in0 disk.drbd2041.in0 disk.drbd1584.out0 cpu13.780224276500494 disk.drbd1584.in0 disk.drbd2111.in0 disk.drbd1965.in0# proc.memory.availableMB38504.5 network.recv593772 disk.drbd2118.out0 disk.drbd1289.in0 disk.drbd1938.in0 disk.dm-0.in 4.64453125 disk.drbd2111.out0 disk.rbd1.in0 disk.drbd1552.in0 disk.drbd1550.in0 disk.drbd1289.out0% memory_percent41.246795654296875 disk.drbd1510.in0 disk.sda.out 132.546875 disk.drbd1226.out0қ2- _wandbruntime4859  _runtime4859c1 _step872 ' lr-AdamW/pg10.00028217966982761647 ' lr-AdamW/pg20.00028217966982761647 trainer/global_step10949 # _timestamp1.78580014678381e+09 _runtime4859.606778305<& ' lr-AdamW/pg10.00028217966982761647 ' lr-AdamW/pg20.00028217966982761647 trainer/global_step10949 # _timestamp1.78580014678381e+09 _runtime4859.606778305 _step872l2- _wandbruntime4859  _runtime48598O YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated192920704808628 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated197615435057152 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated183309448620832 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated74484110 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 9895235932672 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 trainer/global_step10949 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed187980580578304 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9611170384672 % _timestamp1.7858001467847857e+09 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed183309431581472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current260 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed198318989227520 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 188384172 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated197615435057152 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated72745429 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 9634750009856 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed74483848 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 9611256187796 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed187980580578304 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current262 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 9634837439488 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 115629087 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 115638743 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed197615330588160 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated187980597617664 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated72745429 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated43112547 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated187980597617664 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed43112545 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9634750009856 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed72745427 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 115638743 UADeviceStatsMonitor.on_train_batch_sK|KŞ tart/allocated_bytes.all.freed197615330588160 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9611170384672 % _timestamp1.7858001467847857e+09 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed183309431581472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current260 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed198318989227520 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 188384172 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated197615435057152 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated72745429 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 9634750009856 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed74483848 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 9611256187796 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed187980580578304 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current262 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 9634837439488 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 115629087 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 115638743 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed197615330588160 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated187980597617664 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated72745429 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated43112547 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated187980597617664 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed43112545 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9634750009856 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed72745427 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 115638743 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed197615330588160 J6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 188401352 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current256 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 188391680 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 115649280 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9635717066240 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated31374423 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 115639610 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9612133661130 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9635717066240 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 115639610 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated187997764380672 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated74490907 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed188440960411648 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated197633481446912 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9635629629440 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 188401352 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated192938322454282 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 115649280 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed192938219611686 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed197633376970752 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed72752070 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated198337256093184 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 9896270777856 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated187997764380672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 trainer/global_step10949 _runtime4859.981172882 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated43116484 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed183326171753792 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed187997747341312 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated197633481446912 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed74490651 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated183326188793152 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 % _timestamp1.7858001471578352e+09 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800ї>e&M REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated72752072 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed72752070 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9635629629440 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 188401352 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current256 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 188391680 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 115649280 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9635717066240 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated31374423 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 115639610 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9612133661130 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9635717066240 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 115639610 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated187997764380672 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated74490907 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed188440960411648 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated197633481446912 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9635629629440 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 188401352 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated192938322454282 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 115649280 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed192938219611686 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed197633376970752 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed72752070 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated198337256093184 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 9896270777856 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated187997764380672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 trainer/global_step10949 _runtime4859.981172882 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated43116484 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed183326171753792 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed187997747341312 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated197633481446912 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed74490651 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated183326188793152 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 % _timestamp1.7858001471578352e+09 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed31374169 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed43116482 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed198337102619648 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated72752072 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 EDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 116157089 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 189244324 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated43309397 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 9678833574912 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated199224633981952 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated193801601888136 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed193801499045652 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated73077579 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated188838935768064 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 % _timestamp1.7858001691132405e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 _step877 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed74822809 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated188838935768064 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current252 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed199224470015488 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated73077579 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9655058848180 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed43309395 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 9940360880640 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated184146457236832 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9678746145280 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 _runtime 4881.93652649 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9940499943424 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed198517664873984 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 9678746145280 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 189234666 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed73077577 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 9678833574912 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated31513666 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 116166745 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed198517664873984 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current254 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658P>'O JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 116157089 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated74823063 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 9655144651304 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated198517769342976 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed188838918728704 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 trainer/global_step10999 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated198517769342976 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed188838918728704 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated189284134038528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 116157089 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 189244324 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated43309397 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 9678833574912 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated199224633981952 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated193801601888136 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed193801499045652 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated73077579 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated188838935768064 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 % _timestamp1.7858001691132405e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 _step877 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed74822809 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated188838935768064 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current252 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed199224470015488 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated73077579 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9655058848180 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed43309395 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 9940360880640 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated184146457236832 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9678746145280 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 _runtime 4881.93652649 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9940499943424 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed198517664873984 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 9678746145280 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 189234666 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed73077577 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 9678833574912 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated31513666 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 116166745 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed198517664873984 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current254 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current052- _wandbruntime4882  _runtime4882_DM NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed73084220 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed193819116691194 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated198535815732736 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated189301341155328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed188856085491712 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9679713201664 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9679625764864 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed189301316251648 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 116177282 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 trainer/global_step10999 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9679713201664 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 189261504 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated74829783 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated193819219533790 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 189261504 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed73084220 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9679625764864 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9656022124638 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _step878 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated199242740608512 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated198535815732736 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 % _timestamp1.7858001694864476e+09 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed74829543 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated188856102531072 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated184163197409152 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 116167612 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130S*&M JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated193819219533790 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 189261504 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed73084220 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9679625764864 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9656022124638 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _step878 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated199242740608512 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated198535815732736 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 % _timestamp1.7858001694864476e+09 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed74829543 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated188856102531072 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated184163197409152 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 116167612 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed73084220 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed193819116691194 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated198535815732736 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated189301341155328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed188856085491712 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9679713201664 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9679625764864 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed189301316251648 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 116177282 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 trainer/global_step10999 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9679713201664 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 189261504 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated74829783 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2@;i2- _wandbruntime4882  _runtime4882Y trainer/global_step10999 0loss/FRt1/assignment_loss2.8669489132730583 1loss/SRqqt2/assignment_loss0.831187191962575 4loss/FBt1/assignment_loss1.2623771757839327e-06 /loss/FRt2/detection_loss2.3751362562179565 &loss/total_loss15.892389799439155 1loss/SRqqt2/detection_loss0.9809203413893389 /loss/FBt2/detection_loss1.3860402256250381 % _timestamp1.7858001694881032e+09 _runtime4882.310918007 2loss/SRqqt1/assignment_loss0.8068325240561303 1loss/SRqqt1/detection_loss1.6389225296517635 _step879 0loss/FRt2/assignment_loss1.9689547713980617 4loss/FBt2/assignment_loss2.1798135360232403e-06 /loss/FRt1/detection_loss2.4377493433262174 /loss/FBt1/detection_loss0.5996942603483032 epoch0=y 1loss/SRqqt1/detection_loss1.6389225296517635 _step879 0loss/FRt2/assignment_loss1.9689547713980617 4loss/FBt2/assignment_loss2.1798135360232403e-06 /loss/FRt1/detection_loss2.4377493433262174 /loss/FBt1/detection_loss0.5996942603483032 epoch0 trainer/global_step10999 0loss/FRt1/assignment_loss2.8669489132730583 1loss/SRqqt2/assignment_loss0.831187191962575 4loss/FBt1/assignment_loss1.2623771757839327e-06 /loss/FRt2/detection_loss2.3751362562179565 &loss/total_loss15.892389799439155 1loss/SRqqt2/detection_loss0.9809203413893389 /loss/FBt2/detection_loss1.3860402256250381 % _timestamp1.7858001694881032e+09 _runtime4882.310918007 2loss/SRqqt1/assignment_loss0.8068325240561303j~(: ɞU gpu.0.gpu32.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts126.878' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.29266666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0HzV(:  V disk.drbd2111.out0 disk.rbd1.in0 disk.drbd1673.in0 disk.drbd1828.in0 disk.drbd1289.in0 disk.drbd2118.out0 disk.drbd2111.in0 disk.drbd1550.in0 disk.drbd1289.out0# disk.nvme0n1.out224.6474609375 disk.drbd2078.out0 disk.drbd1552.in0 disk.drbd1828.out0 disk.rbd0.in0 disk.drbd1584.out0 disk.sda2.in0 disk.drbd1931.in0! proc.memory.rssMB 11260.40625 disk.rbd0.out0 disk.drbd1510.out0 disk.drbd1965.out0 disk.drbd1938.out0 disk.drbd1584.in0 disk.drbd1673.out0 disk.drbd1931.out0 disk.drbd2041.in0 disk.drbd1552.out0$ memory_percent41.24748110771179 cpu13.780555285807147 disk.drbd1510.in0 disk.sda2.out 133.453125 disk.drbd1965.in0* proc.memory.percent17.182016372680664 disk.drbd1285.out0 disk.nvme0n1.in 4.64453125 disk.drbd1817.in0 disk.rbd1.out 19.79296875 disk.drbd1938.in0 disk.drbd1226.in0 disk.drbd2078.in0* proc.memory.availableMB38504.05078125 disk.drbd1550.out0 disk.dm-0.in 4.64453125 disk.drbd1817.out0 proc.cpu.threads29 disk.drbd1285.in0 disk.dm-0.out224.6474609375 disk.drbd1226.out0* disk./.usagePercent32.344267173593785 disk.drbd1356.in0 disk.drbd1356.out0 network.sent 1.0088382e+07 disk.sda.out 133.453125 disk.sda1.in0 disk.sda1.out0 network.recv597810 disk.drbd2118.in0 disk.drbd2041.out0$ disk./.usageGB578.3483467102051 disk.sda.in0C2- _wandbruntime4904  _runtime4904b2  # _timestamp1.78580019160435e+09 _runtime4904.427327037 _step880 & lr-AdamW/pg10.0002847568884761471 & lr-AdamW/pg20.0002847568884761471 trainer/global_step11049 _step880 & lr-AdamW/pg10.0002847568884761471 & lr-AdamW/pg20.0002847568884761471 trainer/global_step11049 # _timestamp1.78580019160435e+09 _runtime4904.42732703772- _wandbruntime4904  _runtime49048;;'O G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9722742280704 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed199419999159808 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated189697273918464 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 9699033114812 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 % _timestamp1.7858001916053185e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed73409727 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated200130176584704 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed200130023104000 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9698947311688 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 116685091 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9985686706176 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed194682396125160 trainer/global_step11049 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current253 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed189697256879104 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated190144489878528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated194682498967644 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed184983448813472 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated73409729 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 116694747 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 9985558129152 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 116694747 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 116685091 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated73409729 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated43506247 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated199420103628800 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed189697256879104 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 9722829710336 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated189697273918464 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 190104476 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 _step881 _runtime4904.428612375VO JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed199419999159808 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated189697273918464 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 9699033114812 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 % _timestamp1.7858001916053185e+09 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed73409727 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated200130176584704 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed200130023104000 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9698947311688 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 116685091 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated 9985686706176 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed194682396125160 trainer/global_step11049 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current253 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed189697256879104 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated190144489878528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freVed0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated194682498967644 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed184983448813472 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated73409729 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 116694747 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed 9985558129152 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 116694747 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 116685091 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated73409729 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated43506247 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated199420103628800 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed189697256879104 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 9722829710336 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated189697273918464 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 190104476 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 _step881 _runtime4904.428612375 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9722742280704C2- _wandbruntime4904  _runtime4904j&M [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed73416370 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed43510182 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed31659360 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed189714423642112 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9723621900288 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated194700116613298 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9699910588146 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 9986461594624 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9723621900288 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 $ _timestamp1.785800191976226e+09 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated31659617 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 190121656 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated75169801 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 116705284 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed75169542 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 9986590164480 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed185000188985792 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 9699824784910 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated199438150018560 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated190161696995328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 190121656 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated73416372 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 116695614 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated189714440681472 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed194700013770702 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed199438045542400 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated189714440681472 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 190111984 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9723709337088 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated73416372 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 116705284 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated43510184 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated200148287159808 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed190161672091648 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816$ u&M EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed189714423642112 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9723621900288 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated194700116613298 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9699910588146 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed 9986461594624 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9723621900288 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 $ _timestamp1.785800191976226e+09 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated31659617 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 190121656 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated75169801 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 116705284 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed75169542 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated 9986590164480 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed185000188985792 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 9699824784910 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated199438150018560 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated190161696995328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 190121656 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated73416372 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 116695614 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated189714440681472 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed194700013770702 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed199438045542400 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated189714440681472 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 190111984 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9723709337088 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated73416372 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 116705284 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated43510184 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated200148287159808 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed190161672091648 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed73416370 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed43510182 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed31659360 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 =q2- _wandbruntime4904  _runtime4904&Uh 1loss/SRqqt1/detection_loss1.7028553950533802 1loss/SRqqt2/detection_loss0.9584558747431382 .loss/FBt2/detection_loss1.303149523679167 epoch0 0loss/FRt2/assignment_loss2.5369800198568058 2loss/SRqqt1/assignment_loss0.8367024700024298 2loss/FBt2/assignment_loss0.010439198151692253 /loss/FRt2/detection_loss2.6929543381684447 %loss/total_loss17.10968786306043 trainer/global_step11049 % _timestamp1.7858001919778614e+09 _runtime4904.800702659 _step883 /loss/FBt1/detection_loss0.7357628834433854 /loss/FRt1/assignment_loss2.747036758643478 2loss/SRqqt2/assignment_loss0.8443808569234353 3loss/FBt1/assignment_loss0.0014684758416741556 .loss/FRt1/detection_loss2.739502068553398E  1loss/SRqqt1/detection_loss1.7028553950533802 1loss/SRqqt2/detection_loss0.9584558747431382 .loss/FBt2/detection_loss1.303149523679167 epoch0 0loss/FRt2/assignment_loss2.5369800198568058 2loss/SRqqt1/assignment_loss0.8367024700024298 2loss/FBt2/assignment_loss0.010439198151692253 /loss/FRt2/detection_loss2.6929543381684447 %loss/total_loss17.10968786306043 trainer/global_step11049 % _timestamp1.7858001919778614e+09 _runtime4904.800702659 _step883 /loss/FBt1/detection_loss0.7357628834433854 /loss/FRt1/assignment_loss2.747036758643478 2loss/SRqqt2/assignment_loss0.8443808569234353 3loss/FBt1/assignment_loss0.0014684758416741556 .loss/FRt1/detection_loss2.739502068553398KV/B(: U gpu.0.gpu41.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts127.855' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.61833333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0F>X(:  U disk.drbd1828.out0 disk.sda1.in0* proc.memory.percent17.182016372680664 network.recv599375 disk.rbd1.out 19.79296875 disk.drbd1965.out0$ disk.nvme0n1.out233.76904296875 disk.drbd1552.out0 disk.sda.in0 disk.drbd1510.in0 disk.drbd2118.in0 disk.nvme0n1.in 4.64453125 disk.drbd2111.in0 disk.drbd1817.out0 disk.drbd1931.out0 disk.drbd2041.in0 disk.sda.out 133.921875* proc.memory.availableMB38504.05859375 disk.drbd1226.in0 disk.drbd1289.out0 cpu13.781569540969533 disk.sda2.in0 disk.drbd1285.out0 disk.drbd2118.out0 disk.drbd1817.in0 disk.rbd1.in0 disk.drbd1938.in0$ disk./.usageGB578.3487815856934 disk.drbd2078.in0 disk.drbd1584.out0 disk.drbd1828.in0 disk.drbd2041.out0 disk.dm-0.in 4.64453125 disk.drbd2078.out0 disk.drbd1356.in0 disk.drbd1550.out0 disk.sda1.out0! disk.dm-0.out233.76904296875$ memory_percent41.24746918678284 disk.drbd2111.out0 disk.drbd1552.in0 disk.drbd1938.out0 disk.rbd0.in0 proc.cpu.threads29* disk./.usagePercent32.344291494107644 disk.drbd1226.out0 disk.drbd1584.in0 disk.drbd1673.in0 disk.drbd1285.in0 disk.drbd1965.in0 disk.drbd1289.in0 disk.rbd0.out0 disk.drbd1510.out0 network.sent 1.0133375e+07 disk.drbd1931.in0 disk.drbd1550.in0! proc.memory.rssMB 11260.40625 disk.drbd1673.out0 disk.drbd1356.out0 disk.sda2.out 133.921875k&(: U gpu.0.gpu28.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts108.785' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent36.26166666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0rY(:  U disk.drbd1289.in0! disk.dm-0.out234.12060546875 disk.drbd1226.out0! proc.memory.rssMB 11260.40625 network.sent 1.0178424e+07 disk.drbd2111.in0 disk.drbd1817.out0 disk.drbd1356.out0 disk.drbd1552.in0 disk.drbd1552.out0 disk.drbd2078.in0 disk.drbd2118.out0 disk.drbd1285.out0* disk./.usagePercent32.344289360729235$ disk./.usageGB578.3487434387207 disk.sda2.out 134.453125 disk.drbd2041.out0 disk.drbd1938.in0 disk.drbd1938.out0 disk.drbd1285.in0 disk.drbd1510.in0 disk.drbd1965.in0 disk.drbd1673.out0 disk.drbd1931.out0 disk.drbd1828.in0 disk.sda.out 134.453125 disk.sda1.in0$ disk.nvme0n1.out234.12060546875 proc.cpu.threads29 disk.drbd1931.in0 disk.drbd1510.out0 disk.drbd2078.out0$ memory_percent41.24718904495239 cpu13.782503868396907 disk.drbd2118.in0 network.recv600916 disk.drbd1550.in0 disk.drbd1550.out0 disk.rbd0.out0 disk.nvme0n1.in 4.64453125 disk.drbd1584.in0 disk.sda2.in0 disk.rbd1.out 19.9189453125 disk.drbd1673.in0 disk.drbd2041.in0 disk.rbd0.in0 disk.sda1.out0 disk.dm-0.in 4.64453125 disk.drbd1828.out0 disk.drbd1584.out0* proc.memory.percent17.182016372680664 disk.drbd2111.out0 disk.drbd1817.in0 disk.drbd1289.out0 disk.sda.in0 disk.drbd1226.in0) proc.memory.availableMB 38504.2421875 disk.rbd1.in0 disk.drbd1965.out0 disk.drbd1356.in02- _wandbruntime4926  _runtime4926 T trainer/global_step11099 % _timestamp1.7858002139982636e+09 _runtime4926.821212746 _step884 ' lr-AdamW/pg10.00028604549780041234 ' lr-AdamW/pg20.00028604549780041234LS _step884 ' lr-AdamW/pg10.00028604549780041234 ' lr-AdamW/pg20.00028604549780041234 trainer/global_step11099 % _timestamp1.7858002139982636e+09 _runtime4926.8212127462- _wandbruntime4926  _runtime4926eRTO YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated200322437914624 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed200322333445632 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed75502429 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9742835775196 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 ]IDeviceStatsMonitoɅ$r.on_train_batch_start/active_bytes.large_pool.allocated190555612068864 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 190964628 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed73741877 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated31799584 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 9742921578320 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9766738416128 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed195563293204668 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated190555612068864 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed190555595029504 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated10030889737216 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed10030767451648 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated201035735455744 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 trainer/global_step11099 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 9766825845760 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 117213093 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed191004820814848 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 % _timestamp1.7858002139991877e+09 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed190555595029504 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed43703095 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated191004845718528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 _step885 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated75502681 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current252 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated200322437914624 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed200322333445632 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated185820474468832 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed185820457429472 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 117213093 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed31799334 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current250 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed201035588266496 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated195563396047152 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 190954970 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed73741877 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 117222749 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated43703097 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0$'O ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9742835775196 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated190555612068864 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 190964628 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed73741877 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated31799584 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 9742921578320 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9766738416128 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed195563293204668 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated190555612068864 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed190555595029504 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated10030889737216 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed10030767451648 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated201035735455744 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 trainer/global_step11099 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 9766825845760 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 117213093 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed191004820814848 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 % _timestamp1.7858002139991877e+09 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed190555595029504 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed43703095 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated191004845718528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 _step885 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated75502681 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current252 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated200322437914624 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed200322333445632 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated185820474468832 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed185820457429472 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 117213093 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed31799334 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current250 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed201035588266496 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated195563396047152 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 190954970 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed73741877 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 117222749 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated43703097 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated200322437914624 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed200322333445632 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed75502429 60DeviceStatsMonitor.on_train_batch_start/num_ooms0)t 2- _wandbruntime4927  _runtime4927&M EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed75509274 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated73748522 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed43707032 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated185837214641152 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9743799051654 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated10031808408064 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 190972136 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed190572761792512 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9767705472512 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated191022052835328 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed73748520 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated31802492 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed190572761792512 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9767705472512 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime4927.193994047 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 117233286 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated190572778831872 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated75509526 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed191022027931648 _step886 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed200340379828224 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 190972136 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 trainer/global_step11099 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 117223616 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 190981808 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed31802242 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9767618035712 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 % _timestamp1.7858002143706214e+09 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current250 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10031671449600 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated43707034 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712e M E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 190981808 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed31802242 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9767618035712 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 % _timestamp1.7858002143706214e+09 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current250 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10031671449600 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated43707034 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed75509274 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated73748522 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed43707032 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated185837214641152 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9743799051654 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated10031808408064 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 190972136 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed190572761792512 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9767705472512 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated191022052835328 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed73748520 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated31802492 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed190572761792512 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9767705472512 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime4927.193994047 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 117233286 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated190572778831872 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated75509526 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed191022027931648 _step886 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed200340379828224 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 190972136 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 trainer/global_step11099 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 117223616\2- _wandbruntime4927  _runtime4927[9  .loss/FRt1/assignment_loss2.69627360979716 /loss/FRt1/detection_loss2.8161574002106984 /loss/FRt2/detection_loss2.4655653966919466 /loss/FBt1/detection_loss0.5970766416451495 /loss/FBt2/detection_loss1.4312835597277929 _runtime4927.195155635 1loss/SRqqt2/assignment_loss0.945586012000287 epoch0 trainer/global_step11099 _step887 0loss/FRt2/assignment_loss1.9744938682107365 2loss/SRqqt1/assignment_loss0.7095647705191711 1loss/SRqqt2/detection_loss0.8924848501484289 % _timestamp1.7858002143723292e+09 3loss/FBt1/assignment_loss0.0019150208141591105 3loss/FBt2/assignment_loss0.0004144931442523851 1loss/SRqqt1/detection_loss1.7692730483019128 &loss/total_loss16.300088671211697&n] .loss/FRt1/assignment_loss2.69627360979716 /loss/FRt1/detection_loss2.8161574002106984 /loss/FRt2/detection_loss2.4655653966919466 /loss/FBt1/detection_loss0.5970766416451495 /loss/FBt2/detection_loss1.4312835597277929 _runtime4927.195155635 1loss/SRqqt2/assignment_loss0.945586012000287 epoch0 trainer/global_step11099 _step887 0loss/FRt2/assignment_loss1.9744938682107365 2loss/SRqqt1/assignment_loss0.7095647705191711 1loss/SRqqt2/detection_loss0.8924848501484289 % _timestamp1.7858002143723292e+09 3loss/FBt1/assignment_loss0.0019150208141591105 3loss/FBt2/assignment_loss0.0004144931442523851 1loss/SRqqt1/detection_loss1.7692730483019128 &loss/total_loss16.300088671211697u$y(: 㹲U gpu.0.gpu38.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts103.492' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent34.49733333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0 J(:  U network.sent 1.0181441e+07 disk.drbd1550.in0 disk.drbd1285.in0 disk.drbd2118.in0 disk.sda2.out135 disk.drbd2041.in0 disk.sda.out135 disk.drbd1673.in0 disk.rbd0.in0 disk.drbd1356.in0 disk.drbd2111.in0 disk.drbd1673.out0 disk.drbd1828.in0 disk.drbd1938.in0 disk.rbd1.in0 proc.cpu.threads29 disk.drbd1584.out0 disk.drbd2041.out0 disk.drbd1226.in0$ disk.nvme0n1.out236.96533203125 disk.sda1.out0 disk.nvme0n1.in 4.64453125% memory_percent41.246771812438965 disk.drbd1226.out0 disk.drbd2078.in0 disk.drbd2111.out0! proc.memory.rssMB 11260.40625) disk./.usagePercent32.34428509397242 disk.drbd1828.out0 disk.drbd1356.out0 disk.sda1.in0 disk.dm-0.in 4.64453125 disk.sda2.in0 disk.drbd1817.out0 disk.drbd1552.out0 disk.drbd1550.out0 disk.drbd1938.out0 disk.drbd1510.in0 disk.drbd1817.in0 disk.drbd1965.out0( proc.memory.availableMB 38504.515625 disk.drbd1289.out0 disk.rbd0.out0 disk.drbd1285.out0 disk.drbd2078.out0 disk.drbd2118.out0 disk.rbd1.out 19.9189453125 disk.drbd1965.in0 cpu13.783179757973452$ disk./.usageGB578.3486671447754 disk.drbd1552.in0 disk.drbd1289.in0 disk.sda.in0 network.recv601859 disk.drbd1931.in0 disk.drbd1931.out0 disk.drbd1510.out0! disk.dm-0.out236.96533203125 disk.drbd1584.in0* proc.memory.percent17.182016372680664 2- _wandbruntime4948  _runtime4948 & lr-AdamW/pg10.0002873341071246777 & lr-AdamW/pg20.0002873341071246777 trainer/global_step11149 $ _timestamp1.785800235686339e+09 _runtime4948.509299094 _step888pUl+ & lr-AdamW/pg10.0002873341071246777 & lr-AdamW/pg20.0002873341071246777 trainer/global_step11149 $ _timestamp1.785800235686339e+09 _runtime4948.509299094 _step888&)2- _wandbruntime4948  _runtime4948\րj'O E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed196444190284176 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed74074027 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 120188416 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed191413933179904 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed186657466045472 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 _runtime4948.510588988 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 % _timestamp1.7858002356873598e+09 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 117741095 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated191413950219264 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed191413933179904 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated196444293126660 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed31943175 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated201941301858304 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 9786810041828 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 9810821981184 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 trainer/global_step11149 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated75843364 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 191824780 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed201224667731456 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 9810821981184 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated31943417 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 _step889 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated191865201558528 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed191865176654848 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 191815122 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated10076100299776 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed74074027 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9786724238704 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated201224772200448 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated74074029 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 117750751 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed201224667731456 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated201224772200448 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 145092096 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121'P'O ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 9810734551552 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed75843120 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed201941156766208 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed74074027 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9786724238704 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated201224772200448 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated74074029 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 117750751 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed201224667731456 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated201224772200448 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 145092096 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed196444190284176 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed74074027 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 120188416 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed191413933179904 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed186657466045472 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 _runtime4948.510588988 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 % _timestamp1.7858002356873598e+09 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 117741095 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated191413950219264 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed191413933179904 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated196444293126660 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed31943175 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated201941301858304 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 9786810041828 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 9810821981184 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 trainer/global_step11149 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated75843364 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 191824780 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed201224667731456 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 9810821981184 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated31943417 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 _step889 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated191865201558528 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed191865176654848 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 191815122 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated10076100299776 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 JDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 % _timestamp1.7858002360620434e+09 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed196461807929718 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 191832288 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed74080670 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.pel1!ak 2712961024 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed191882383771648 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed191431099942912 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 117751618 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated196461910772314 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated75850163 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9811614171136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated31946279 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 _runtime4948.885414688 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated201242818590208 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed201242714114048 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 trainer/global_step11149 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated186674223257152 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed43903882 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 191841960 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed31946025 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9811614171136 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 117761288 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10076875950080 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed75849907 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current254 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 117761288 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated43903884 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 9787601711926 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed186674206217792 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated201242818590208 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated74080672 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated191431116982272 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 191832288 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated74080672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed06|&M WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated201242818590208 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated74080672 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated191431116982272 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 191832288 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated74080672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed74080670 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 117751618 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current256 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9787687515162 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 % _timestamp1.7858002360620434e+09 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed196461807929718 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 191832288 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed74080670 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed191882383771648 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed191431099942912 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 117751618 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated196461910772314 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated75850163 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9811614171136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated31946279 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 _runtime4948.885414688 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated201242818590208 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed201242714114048 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 trainer/global_step11149 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated186674223257152 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed43903882 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 191841960 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed31946025 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9811614171136 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 117761288 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10076875950080 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed75849907 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current254 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 117761288 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated43903884 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 9787601711926 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed186674206217792 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640_M2- _wandbruntime4948  _runtime4948Hi epoch0 0loss/FRt1/assignment_loss2.4642511845749113 /loss/FRt1/detection_loss2.2274163468418835 1loss/SRqqt2/detection_loss0.9636455208339073 trainer/global_step11149 % _timestamp1.7858002360637107e+09 1loss/FBt1/assignment_loss0.06390649718939301 .loss/FBt2/detection_loss1.733121257293253 _runtime4948.886546016 /loss/FRt2/assignment_loss2.430450118382772 2loss/SRqqt2/assignment_loss0.8722286351576999 3loss/FBt2/assignment_loss0.0027638033326624623 .loss/FRt2/detection_loss2.442156935532888 /loss/FBt1/detection_loss1.2871183022258006 _step891 2loss/SRqqt1/assignment_loss0.8644792924663334 1loss/SRqqt1/detection_loss1.5239365332280144 &loss/total_loss16.875474427059515ګ  /loss/FRt2/assignment_loss2.430450118382772 2loss/SRqqt2/assignment_loss0.8722286351576999 3loss/FBt2/assignment_loss0.0027638033326624623 .loss/FRt2/detection_loss2.442156935532888 /loss/FBt1/detection_loss1.2871183022258006 _step891 2loss/SRqqt1/assignment_loss0.8644792924663334 1loss/SRqqt1/detection_loss1.5239365332280144 &loss/total_loss16.875474427059515 epoch0 0loss/FRt1/assignment_loss2.4642511845749113 /loss/FRt1/detection_loss2.2274163468418835 1loss/SRqqt2/detection_loss0.9636455208339073 trainer/global_step11149 % _timestamp1.7858002360637107e+09 1loss/FBt1/assignment_loss0.06390649718939301 .loss/FBt2/detection_loss1.733121257293253 _runtime4948.886546016K(: U gpu.0.gpu22.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts126.478' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.15933333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0Ұi](:  U disk.drbd1965.in0 disk.drbd2041.out0 disk.sda.in0! disk.dm-0.out237.18798828125 disk.drbd2118.in0 disk.drbd1673.in0 disk.rbd0.out0 disk.drbd1226.in0 disk.sda2.in0 disk.rbd1.in0 disk.drbd1289.out0 disk.rbd0.in0 disk.sda.out 135.26171875 disk.drbd2118.out0) disk./.usagePercent32.34428829404003 disk.sda1.out0 disk.drbd1828.out0 disk.drbd1510.in0 disk.drbd1965.out0 disk.sda1.in0 disk.drbd1584.out0! proc.memory.rssMB 11260.40625 disk.drbd1289.in0 network.sent 1.0226405e+07 disk.drbd1817.out0 disk.drbd1931.in0 disk.drbd1931.out0 disk.drbd1285.out0 disk.drbd1510.out0 disk.dm-0.in 4.64453125* proc.memory.availableMB38504.48828125$ disk./.usageGB578.3487243652344 disk.drbd1356.in0 proc.cpu.threads29 disk.drbd2111.in0 disk.drbd2111.out0 disk.drbd1828.in0 disk.drbd1356.out0 network.recv603244 disk.drbd2078.in0 disk.drbd2078.out0 disk.drbd1584.in0 disk.drbd1817.in0 disk.rbd1.out 19.9814453125 disk.drbd1552.out0 disk.nvme0n1.in 4.64453125 disk.sda2.out 135.26171875 disk.drbd1673.out0 disk.drbd2041.in0 disk.drbd1938.out0$ disk.nvme0n1.out237.18798828125$ memory_percent41.24681353569031 cpu13.782607852738558 disk.drbd1552.in0 disk.drbd1550.out0 disk.drbd1285.in0 disk.drbd1226.out0* proc.memory.percent17.182016372680664 disk.drbd1550.in0 disk.drbd1938.in0ҍK(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts102.402' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent34.134 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0~mX(:  U disk.drbd2041.out0 disk.drbd1550.in0 disk.drbd1289.in0 disk.drbd1817.out0 disk.drbd1552.in0! proc.memory.rssMB 11260.40625 disk.sda1.in0 disk.sda2.in0 disk.drbd1817.in0 disk.drbd1552.out0 disk.drbd1938.out0 network.sent 1.0271409e+07 disk.drbd1285.out0! disk.nvme0n1.out 242.60546875 disk.rbd0.in0 disk.sda.out 135.49609375 disk.drbd1510.out0 disk.drbd1931.in0 disk.drbd1828.out0 disk.drbd1226.in0 network.recv604804 disk.drbd1285.in0 cpu13.781964394088181 disk.sda.in0 disk.drbd1965.in0 disk.drbd2041.in0$ memory_percent41.24719500541687 disk.rbd0.out0 disk.drbd1356.out0 disk.sda2.out 135.49609375 disk.drbd1828.in0 disk.drbd1938.in0 disk.dm-0.out 242.60546875 disk.drbd1584.in0 disk.drbd2118.out0 disk.rbd1.in0 disk.rbd1.out 19.9814453125 proc.cpu.threads29 disk.drbd2111.in0 disk.drbd1550.out0 disk.drbd1289.out0* disk./.usagePercent32.344289360729235 disk.nvme0n1.in 4.64453125 disk.drbd2111.out0 disk.drbd2118.in0 disk.drbd1673.in0 disk.drbd1931.out0* proc.memory.availableMB38504.23828125$ disk./.usageGB578.3487434387207 disk.drbd1510.in0 disk.drbd2078.in0 disk.drbd2078.out0 disk.drbd1673.out0* proc.memory.percent17.182016372680664 disk.drbd1584.out0 disk.dm-0.in 4.64453125 disk.sda1.out0 disk.drbd1226.out0 disk.drbd1356.in0 disk.drbd1965.out0 U2- _wandbruntime4971  _runtime4971lP6 ' lr-AdamW/pg20.00028862271644894297 trainer/global_step11199 % _timestamp1.7858002581824222e+09 _runtime4971.005396029 _step892 ' lr-AdamW/pg10.00028862271644894297 ' lr-AdamW/pg10.00028862271644894297 ' lr-AdamW/pg20.00028862271644894297 trainer/global_step11199 % _timestamp1.7858002581824222e+09 _runtime4971.005396029 _step892uZ2- _wandbruntime4971  _runtime4971Y˔O trainer/global_step11199 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9854730686976 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 192684932 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated197325190206168 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed192272271330304 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed74406177 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 % _timestamp1.7858002581834004e+09 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed202127002017280 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 118278753 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current236 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed192272271330304 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current238 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated192272288369664 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated76183565 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated44096797 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed44096795 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 UADeviceb! StatsMonitor.on_train_batch_start/allocated_bytes.all.freed202127002017280 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 9854818116608 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed32086532 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed202846663358976 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 9854730686976 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed202127002017280 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 118278753 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current236 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed192272271330304 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current238 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated192272288369664 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated76183565 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated44096797 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed44096795 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed202127002017280 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 9854818116608 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed32086532 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed202846663358976 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 9854730686976 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 _step894 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 9831490175434 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed192289438093312 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed202145048399872 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated192289455132672 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 118279620 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current243 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10122039597056 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed187511214833792 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated192289455132672 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 192692440 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 192692440 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current245 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated202864936876544 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 192702112 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated32089652 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated44100734 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 118279620 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed197342705009226 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed74412820 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed76190141 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9831575978670 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed44100732 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed74412820 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed202864779208704 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed192742739611648 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated202145152876032 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated197342807851822 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251B&M NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed74412820 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed202864779208704 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed192742739611648 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated202145152876032 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated197342807851822 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 _step894 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 9831490175434 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed192289438093312 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed202145048399872 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated192289455132672 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 118279620 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current243 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10122039597056 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed187511214833792 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated192289455132672 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 192692440 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 192692440 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current245 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated202864936876544 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 192702112 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated32089652 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated44100734 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 118279620 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed197342705009226 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed74412820 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed76190141 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9831575978670 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed44100732 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360@|2- _wandbruntime4971  _runtime4971 % _timestamp1.7858002585719638e+09 _runtime4971.394786936 _step895 1loss/FBt2/assignment_loss0.08722158035465434 1loss/SRqqt2/detectiͭcon_loss1.0630961438444884 epoch0 trainer/global_step11199 0loss/FRt2/assignment_loss2.7025647878646852 2loss/SRqqt1/assignment_loss1.1114132802086798 2loss/SRqqt2/assignment_loss0.9688960335854637 2loss/FBt1/assignment_loss0.009961298747290386 /loss/FRt1/assignment_loss2.547460675239563 .loss/FRt2/detection_loss2.706751048352037 /loss/FBt2/detection_loss1.7595267430468298 %loss/total_loss17.87198271417465 .loss/FRt1/detection_loss2.353870551399094 1loss/SRqqt1/detection_loss1.6846165831793438 /loss/FBt1/detection_loss0.8766039883525207H  .loss/FRt1/detection_loss2.353870551399094 1loss/SRqqt1/detection_loss1.6846165831793438 /loss/FBt1/detection_loss0.8766039883525207 % _timestamp1.7858002585719638e+09 _runtime4971.394786936 _step895 1loss/FBt2/assignment_loss0.08722158035465434 1loss/SRqqt2/detection_loss1.0630961438444884 epoch0 trainer/global_step11199 0loss/FRt2/assignment_loss2.7025647878646852 2loss/SRqqt1/assignment_loss1.1114132802086798 2loss/SRqqt2/assignment_loss0.9688960335854637 2loss/FBt1/assignment_loss0.009961298747290386 /loss/FRt1/assignment_loss2.547460675239563 .loss/FRt2/detection_loss2.706751048352037 /loss/FBt2/detection_loss1.7595267430468298 %loss/total_loss17.871982714174657Ϛ(: U gpu.0.gpu36.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts96.926' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent32.30866666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0N(:  U disk.drbd1965.in0 disk.rbd0.in0 disk.sda1.in0 disk.drbd1938.out0 disk.drbd1356.in0* proc.memory.percent17.182016372680664 proc.cpu.threads29$ disk./.usageGB578.3492126464844 disk.drbd1510.in0 disk.drbd1226.out0 cpu13.781151932411095 disk.rbd0.out0 disk.sda1.out0 disk.dm-0.out 242.90625( proc.memory.availableMB 38504.421875 disk.drbd2041.out0 disk.sda.out 135.9609375 disk.drbd2111.in0 disk.drbd2111.out0 disk.drbd1817.in0 disk.drbd1965.out0 disk.sda2.in0 disk.drbd1673.in0 disk.drbd1931.out0 disk.drbd1828.in0 disk.drbd1289.out0 disk.drbd2078.in0 disk.drbd2078.out0 disk.drbd1584.in0 disk.sda2.out 135.9609375 disk.drbd1550.in0 disk.drbd1584.out0 disk.drbd2118.out0 disk.rbd1.in0 network.sent 1.0274416e+07 disk.drbd1552.in0 disk.drbd1285.in0 disk.drbd2041.in0 disk.drbd1550.out0 disk.drbd1289.in0 disk.drbd2118.in0% memory_percent41.246914863586426 disk.drbd1828.out0 disk.nvme0n1.in 4.64453125 disk.drbd1817.out0 disk.drbd1552.out0 disk.sda.in0 disk.drbd1285.out0 disk.rbd1.out 20.107421875! proc.memory.rssMB 11260.40625 disk.drbd1931.in0 disk.drbd1938.in0 disk.drbd1510.out0 disk.drbd1226.in0 disk.nvme0n1.out 242.90625 network.recv605702* disk./.usagePercent32.344315601283654 disk.drbd1673.out0 disk.dm-0.in 4.64453125 disk.drbd1356.out0^a2- _wandbruntime4993  _runtime4993F _runtime4993.128655588 _step896 & lr-AdamW/pg10.0002899113257732083 & lr-AdamW/pg20.0002899113257732083 trainer/global_step11249 % _timestamp1.7858002803056862e+09Z % _timestamp1.7858002803056862e+09 _runtime4993.128655588 _step896 & lr-AdamW/pg10.0002899113257732083 & lr-AdamW/pg20.0002899113257732083 trainer/global_step11249Ǩ2- _wandbruntime4993  _runtime4993_'O G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed203029336303104 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 9874586968844 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 _runtime4993.129946884 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9874501165720 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated76523550 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed44293645 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated32229903 trainer/global_step11249 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 118797099 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated198206087285676 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed193130609480704 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 193545084 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0y'O QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed193130609480704 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed203029336303104 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 9874586968844 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 _runtime4993.129946884 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9874501165720 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated76523550 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed44293645 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated32229903 trainer/global_step11249 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 118797099 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated198206087285676 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed193130609480704 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 193545084 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _step898 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated193147793283072 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed193147776243712 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 193552592 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated74744972 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed74744970 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 trainer/global_step11249 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 118817292 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed32232617 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated188348240489152 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated203770490499072 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated10167370143744 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated32232851 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated203047487161856 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 193562264 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated193603120355328 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated198223704931330 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated193147793283072 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated76530435 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current236 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed44297582 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current234 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 9875378638942 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed198223602088734 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 _runtime4993.500145154 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 118807622 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9899693878784 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9875464442178 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 118807622 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated44297584 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9899606441984 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 118817292 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10167241573888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 193562264 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated74744972 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9899693878784 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed76530199 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032e&o&M OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated203770490499072 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated10167370143744 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated32232851 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated203047487161856 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 193562264 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated193603120355328 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated198223704931330 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated193147793283072 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated76530435 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current236 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed44297582 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current234 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 9875378638942 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed198223602088734 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 _runtime4993.500145154 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 118807622 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9899693878784 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9875464442178 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 118807622 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated44297584 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9899606441984 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 118817292 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10167241573888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 193562264 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated74744972 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9899693878784 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed76530199 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed203047382685696 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _step898 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated193147793283072 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed193147776243712 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 193552592 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated74744972 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed74744970 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 trainer/global_step11249 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 118817292 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed32232617 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated188348240489152D}2- _wandbruntime4993  _runtime4993%v /loss/FRt2/detection_loss2.3766112069735366 0loss/SRqqt1/detection_loss1.566912111790421 1loss/SRqqt2/detection_loss1.1102872721318688 % _timestamp1.7858002806784334e+09 _step899 2loss/SRqqt1/assignment_loss1.0126182166366982 /loss/FRt1/detection_loss2.4780254888383646 &loss/total_loss16.914929326191036 epoch0 trainer/global_step11249 _runtime4993.501259567 /loss/FBt2/detection_loss1.5448493685301703 /loss/FRt1/assignment_loss2.467048414145844 0loss/FRt2/assignment_loss2.5018287441703713 3loss/FBt1/assignment_loss0.0019164687367551228 1loss/FBt2/assignment_loss0.08972812959170026 .loss/FBt1/detection_loss0.789763233539733 2loss/SRqqt2/assignment_loss0.9753406711055764#  1loss/SRqqt2/detection_loss1.1102872721318688 % _timestamp1.7858002806784334e+09 _step899 2loss/SRqqt1/assignment_loss1.0126182166366982 /loss/FRt1/detection_loss2.4780254888383646 &loss/total_loss16.914929326191036 epoch0 trainer/global_step11249 _runtime4993.501259567 /loss/FBt2/detection_loss1.5448493685301703 /loss/FRt1/assignment_loss2.467048414145844 0loss/FRt2/assignment_loss2.5018287441703713 3loss/FBt1/assignment_loss0.0019164687367551228 1loss/FBt2/assignment_loss0.08972812959170026 .loss/FBt1/detection_loss0.789763233539733 2loss/SRqqt2/assignment_loss0.9753406711055764 /loss/FRt2/detection_loss2.3766112069735366 0loss/SRqqt1/detection_loss1.566912111790421j W(: U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts124.952' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent41.650666666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0kT(:  ׀V disk.drbd1817.out0 disk.drbd1828.out0 disk.drbd2111.in0 disk.drbd1673.in0 disk.drbd1552.out0 disk.sda.in0$ disk./.usageGB578.3491706848145 disk.drbd1965.in0 disk.drbd2041.out0 disk.drbd1356.out0 disk.drbd1550.in0 disk.sda1.in0 disk.dm-0.in 4.64453125 network.sent 1.0319598e+07 disk.drbd1552.in0 disk.drbd1289.out0 disk.drbd1938.out0 disk.rbd0.out0 disk.drbd2078.in0* proc.memory.percent17.182016372680664 disk.drbd1356.in0 disk.drbd1931.in0 proc.cpu.threads29 disk.drbd1965.out0 disk.drbd1226.in0 disk.drbd1584.in0% memory_percent41.246795654296875 network.recv607207 disk.sda.out 136.21484375# proc.memory.availableMB38504.5 disk.rbd1.out 20.107421875 disk.drbd1828.in0 disk.drbd1550.out0 disk.drbd1938.in0 disk.rbd1.in0 disk.rbd0.in0 disk.sda1.out0 disk.drbd1285.in0 disk.drbd1285.out0 disk.drbd2078.out0 disk.drbd2118.out0 disk.drbd1510.in0 disk.dm-0.out247.2412109375 disk.drbd1584.out0 disk.drbd2111.out0# disk.nvme0n1.out247.2412109375 disk.sda2.in0 disk.nvme0n1.in 4.64453125 disk.drbd1226.out0 disk.drbd1673.out0 disk.drbd1931.out0 disk.drbd2041.in0 disk.sda2.out 136.21484375) disk./.usagePercent32.34431325456741 disk.drbd2118.in0 disk.drbd1817.in0 disk.drbd1289.in0 disk.drbd1510.out0! proc.memory.rssMB 11260.40625 cpu13.780538880954532:&(: U gpu.0.gpu36.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts126.584' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.19466666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0_[(:  U disk.sda1.out0 disk.drbd1584.out0 disk.drbd1817.out0 network.sent 1.0364701e+07 disk.drbd1550.out0 disk.dm-0.in 4.64453125 disk.drbd1289.in0 disk.drbd2111.in0 disk.rbd0.out0 disk.drbd1356.in0 disk.drbd1965.in0 disk.drbd1931.in0 disk.drbd2041.in0 disk.drbd1938.in0 disk.sda2.out 136.54296875 disk.drbd1828.out0 disk.drbd1584.in0 disk.drbd2118.in0 disk.drbd1285.out0 disk.drbd2118.out0 disk.drbd1931.out0 disk.drbd1552.in0 disk.drbd1226.out0 disk.nvme0n1.in 4.64453125 disk.drbd1510.in0# disk.nvme0n1.out247.5185546875* disk./.usagePercent32.344543019422005 disk.drbd1289.out0 disk.drbd2111.out0 disk.drbd1965.out0 cpu13.779907330211609 proc.cpu.threads29 disk.drbd2041.out0 disk.sda1.in0 disk.drbd1356.out0 disk.rbd1.in0 network.recv608932 disk.drbd1552.out0 disk.drbd1938.out0 disk.drbd1510.out0 disk.drbd2078.out0 disk.drbd1673.in0 disk.drbd1673.out0 disk.drbd1550.in0 disk.rbd0.in0 disk.sda.in0 disk.dm-0.out247.5185546875$ memory_percent41.24687314033508! proc.memory.rssMB 11260.40625$ disk./.usageGB578.3532791137695 disk.sda.out 136.54296875 disk.drbd1226.in0 disk.drbd1817.in0* proc.memory.availableMB38504.44921875* proc.memory.percent17.182016372680664 disk.drbd1285.in0 disk.sda2.in0 disk.drbd1828.in0 disk.drbd2078.in0 disk.rbd1.out 20.1699218752- _wandbruntime5015  _runtime5015$Ϭ _runtime5015.178710065 _step900 & lr-AdamW/pg10.0002911999350974736 & lr-AdamW/pg20.0002911999350974736 trainer/global_step11299 % _timestamp1.7858003023557422e+09Yt trainer/global_step11299 % _timestamp1.7858003023557422e+09 _runtime5015.178710065 _step900 & lr-AdamW/pg10.0002911999350974736 & lr-AdamW/pg20.0002911999350974736k2- _wandbruntime5015  _runtime5015xG'O `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated189168508932832 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed193988947631104 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed189168491893472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 9942722957824 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated75070479 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated203931775057920 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed76863793 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated193988964670464 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 119334757 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed75070477 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed203931670588928 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 9942810387456 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed75070477 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed10211548657152 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _runtime5015.180004592 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated199086984365184 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 9918475432352 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated32373570 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated193988964670464 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed203931670588928 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed193988947631104 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed204657792832000 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 % _timestamp1.7858003023567097e+09 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated44490497 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 119325101 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 9942810387456 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 119325101 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed194446244174848 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 119334757 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated203931775057920 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated75070479 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9942722957824 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated204657956798464 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 _step901 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 194405236 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 trainer/global_step11299 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed44490495 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated10211687719936 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed32373298P9#CO LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated 9918475432352 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated32373570 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated193988964670464 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed203931670588928 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed193988947631104 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed204657792832000 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 % _timestamp1.7858003023567097e+09 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated44490497 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 119325101 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated 9942810387456 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 119325101 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed194446244174848 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 119334757 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated203931775057920 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated75070479 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 GADeviceStatsMonitor.on_train_batch_staVKޛrt/active.large_pool.current2 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9942722957824 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated204657956798464 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 _step901 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 194405236 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 trainer/global_step11299 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed44490495 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated10211687719936 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed32373298 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated189168508932832 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed193988947631104 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed189168491893472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed 9942722957824 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated75070479 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated203931775057920 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed76863793 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated193988964670464 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 119334757 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed75070477 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed203931670588928 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated 9942810387456 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed75070477 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed10211548657152 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _runtime5015.180004592 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated1990869843651842- _wandbruntime5015  _runtime50150&M H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed76870621 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9943602577408 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9943690014208 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10212457144320 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 _runtime 5015.55978042 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current246 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated10212587811328 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated44494434 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated75077122 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 119345294 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated199104602010838 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9943602577408 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed194006114394112 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current248 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed75077120 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 9919267102450 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 194422416 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed189185232065792 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 % _timestamp1.7858003027364328e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 _step902 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated189185249105152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 119335624 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed32376189 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed194463451291648 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 trainer/global_step11299 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated203949821447680 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed203949716971520 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed204675908435968 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 194412744 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed199104499168242 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated194463476195328 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated76870869 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9943690014208 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed75077120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated194006131433472 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated32376435 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated203949821447680 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated194006131433472 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 119345294 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 119335624 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated204676064006656 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9919352905686 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated76870869 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated 9943690014208 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed75077120 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated194006131433472 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated32376435 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated203949821447680 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated194006131433472 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 119345294 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 119335624 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated204676064006656 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated 9919352905686 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed76870621 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed 9943602577408 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9943690014208 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10212457144320 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 _runtime 5015.55978042 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current246 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated10212587811328 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated44494434 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated75077122 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 119345294 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated199104602010838 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9943602577408 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed194006114394112 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current248 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed75077120 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 9919267102450 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 194422416 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed189185232065792 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 % _timestamp1.7858003027364328e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 _step902 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated189185249105152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 119335624 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed32376189 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed194463451291648 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 trainer/global_step11299 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated203949821447680s2- _wandbruntime5015  _runtime5015H[  .loss/FBt1/detection_loss0.705603999362604 _step903 /loss/FBt2/detection_loss1.0813477939138045 epoch0 0loss/FRt2/assignment_loss2.3488309677441914 1loss/SRqqt2/assignment_loss0.980259867637263 4loss/FBt1/assignment_loss0.00010885069196575614 trainer/global_step11299 % _timestamp1.7858003027381258e+09 .loss/FRt1/assignment_loss2.86236899373043 3loss/FBt2/assignment_loss4.584972183115846e-08 /loss/FRt2/detection_loss2.5042678827047347 &loss/total_loss16.573347220786665 _runtime5015.560962763 2loss/SRqqt1/assignment_loss0.7215163270011544 /loss/FRt1/detection_loss2.4660177006419883 1loss/SRqqt1/detection_loss1.9196945410358774 1loss/SRqqt2/detection_loss0.9833302504729289i /loss/FBt2/detection_loss1.0813477939138045 epoch0 0loss/FRt2/assignment_loss2.3488309677441914 1loss/SRqqt2/assignment_loss0.980259867637263 4loss/FBt1/assignment_loss0.00010885069196575614 trainer/global_step11299 % _timestamp1.7858003027381258e+09 .loss/FRt1/assignment_loss2.86236899373043 3loss/FBt2/assignment_loss4.584972183115846e-08 /loss/FRt2/detection_loss2.5042678827047347 &loss/total_loss16.573347220786665 _runtime5015.560962763 2loss/SRqqt1/assignment_loss0.7215163270011544 /loss/FRt1/detection_loss2.4660177006419883 1loss/SRqqt1/detection_loss1.9196945410358774 1loss/SRqqt2/detection_loss0.9833302504729289 .loss/FBt1/detection_loss0.705603999362604 _step903e(: U gpu.0.gpu0.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts127.337' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.44566666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0y ](:  U disk.drbd1965.in0 disk.drbd1552.out0 disk.drbd1938.in0 disk.drbd2111.in0 disk.drbd1828.out0 disk.drbd1510.out0 network.recv609952 disk.rbd1.in0 disk.drbd1965.out0 disk.sda.in0 disk.drbd1356.out0 disk.sda2.out 136.78515625 disk.drbd1931.out0! disk.dm-0.out247.79638671875 disk.drbd2078.in0 disk.drbd1584.in0 network.sent 1.0367823e+07 disk.drbd1817.in0 disk.drbd1552.in0 disk.sda.out 136.78515625 disk.sda1.in0 disk.drbd2118.in0$ disk.nvme0n1.out247.79638671875$ disk./.usageGB578.3577156066895 disk.drbd1550.in0 disk.rbd0.in0 disk.sda1.out0 disk.drbd1510.in0 disk.drbd1226.in0 cpu13.779132965823184 disk.drbd2111.out0 disk.drbd1550.out0 disk.drbd1938.out0 disk.rbd0.out0 disk.drbd1285.out0 disk.drbd2118.out0* disk./.usagePercent32.344791131330915 disk.sda2.in0 disk.dm-0.in 4.64453125 disk.drbd1584.out0$ memory_percent41.24653935432434 disk.drbd1673.out0 disk.drbd2078.out0 disk.nvme0n1.in 4.64453125 disk.drbd1817.out0 disk.drbd1285.in0 disk.drbd1356.in0 disk.drbd1673.in0 disk.drbd1931.in0 disk.drbd1289.out0 disk.drbd1226.out0 proc.cpu.threads29 disk.drbd1828.in0 disk.drbd1289.in0* proc.memory.percent17.182016372680664! proc.memory.rssMB 11260.40625 disk.drbd2041.in0 disk.rbd1.out 20.169921875 disk.drbd2041.out0* proc.memory.availableMB38504.66796875'2- _wandbruntime5037  _runtime5037-g _step904 & lr-AdamW/pg10.0002924885444217389 & lr-AdamW/pg20.0002924885444217389 trainer/global_step11349 % _timestamp1.7858003247001672e+09 _runtime5037.523120754 &Em _runtime5037.523120754 _step904 & lr-AdamW/pg10.0002924885444217389 & lr-AdamW/pg20.0002924885444217389 trainer/global_step11349 % _timestamp1.7858003247001672e+09O>2- _wandbruntime5037  _runtime5037`` O VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated204834109343744 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed204834004874752 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current260 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9962278092736 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 119853103 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated44687347 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated75402629 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9986719093248 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated77205223 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated199967881444692 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 % _timestamp1.7858003247011006e+09 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_po vq ol.current87429632 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed199967778602208 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated190005517548832 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed205563322137088 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated204834109343744 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed44687345 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 195265388 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed75402627 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed195306600014848 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 _step905 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated194847302820864 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed194847285781504 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed190005500509472 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated75402629 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed75402627 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 119862759 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed77204963 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 trainer/global_step11349U@'O N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 195265388 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed75402627 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed195306600014848 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 _step905 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated194847302820864 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed194847285781504 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed190005500509472 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated75402629 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed75402627 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 119862759 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed77204963 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 trainer/global_step11349 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated204834109343744 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed204834004874752 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current260 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed 9962278092736 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 119853103 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated44687347 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated75402629 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed 9986719093248 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated77205223 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated199967881444692 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 % _timestamp1.7858003247011006e+09 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed199967778602208 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated190005517548832 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed205563322137088 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated204834109343744 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed44687345 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0$2- _wandbruntime5037  _runtime5037e2n&M bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated10257760629248 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated75409272 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 119863626 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed 9987598712832 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed77211777 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 195282568 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated32520750 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated75409272 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed75409270 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current257 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated190022257721152 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 119863626 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed204852051257344 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed194864452544512 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 119873296 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated204852155733504 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9987686149632 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10257619476480 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 _step906 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 141152768 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 trainer/global_step11349 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 195272896 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed195323807131648 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated194864469583872 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed199985396247750 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 195272896 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated77212034 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed75409270 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated194864469583872 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 195282568 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed190022240681792 _runtime5037.891581363 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed 9963155565958 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 119873296 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current255 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 119873296 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current255 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed77211777 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 195282568 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated32520750 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated75409272 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed75409270 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current257 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 H?Devib4 ceStatsMonitor.on_train_batch_end/active.small_pool.current9670 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated190022257721152 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 119863626 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed204852051257344 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed194864452544512 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 119873296 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated204852155733504 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated 9987686149632 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10257619476480 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 _step906 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 141152768 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 trainer/global_step11349 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 195272896 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed195323807131648 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated194864469583872 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed199985396247750 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 195272896 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated77212034 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed75409270 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated194864469583872 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 195282568 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed190022240681792 _runtime5037.891581363 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak2516Z2- _wandbruntime5037  _runtime5037ƒr 2loss/SRqqt1/assignment_loss0.9691548641164636 4loss/FBt1/assignment_loss1.2772421850318746e-08 &loss/total_loss15.749710420139941 trainer/global_step11349 _runtime5037.892679235 4loss/FBt2/assignment_loss3.9168734084731635e-07 /loss/FRt1/detection_loss2.4247903959618675 .loss/FRt2/detection_loss2.246711607162769 /loss/FBt2/detection_loss0.8214183042624167 % _timestamp1.7858003250698504e+09 0loss/FRt1/assignment_loss2.9066157705254025 2loss/SRqqt2/assignment_loss0.8697030716920009 1loss/SRqqt2/detection_loss1.0179282057977899 /loss/FBt1/detection_loss0.4430937077184873 epoch0 _step907 0loss/FRt2/assignment_loss2.1048290248100576 1loss/SRqqt1/detection_loss1.9454650636329207 ƈ 2loss/SRqqt1/assignment_loss0.9691548641164636 4loss/FBt1/assignment_loss1.2772421850318746e-08 &loss/total_loss15.749710420139941 trainer/global_step11349 _runtime5037.892679235 4loss/FBt2/assignment_loss3.9168734084731635e-07 /loss/FRt1/detection_loss2.4247903959618675 .loss/FRt2/detection_loss2.246711607162769 /loss/FBt2/detection_loss0.8214183042624167 % _timestamp1.7858003250698504e+09 0loss/FRt1/assignment_loss2.9066157705254025 2loss/SRqqt2/assignment_loss0.8697030716920009 1loss/SRqqt2/detection_loss1.0179282057977899 /loss/FBt1/detection_loss0.4430937077184873 epoch0 _step907 0loss/FRt2/assignment_loss2.1048290248100576 1loss/SRqqt1/detection_loss1.9454650636329207a(: ЋU gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts100.644' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent33.548 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0|#X(:  ؄V disk.rbd1.in0! proc.memory.rssMB 11260.40625 cpu13.778580533422337 disk.drbd1550.in0 disk.drbd1938.in0 disk.rbd0.in0 disk.drbd1965.out0 disk.drbd2041.out0 disk.drbd1510.out0 disk.dm-0.in 4.64453125 disk.drbd2118.out0$ memory_percent41.24798774719238$ disk.nvme0n1.out248.59326171875 disk.sda2.out 137.23828125 disk.drbd2041.in0 proc.cpu.threads29 disk.nvme0n1.in 4.64453125 disk.drbd2111.in0 disk.drbd1552.in0 disk.drbd1584.in0 disk.drbd1817.in0 disk.drbd1552.out0 disk.rbd0.out0 disk.drbd1226.in0 disk.sda.in0 network.recv611349! disk.dm-0.out248.59326171875 disk.drbd1817.out0 disk.drbd1931.in0) disk./.usagePercent32.34481523850693 disk.drbd1285.out0 disk.drbd1510.in0 disk.drbd2118.in0 disk.drbd2111.out0 disk.rbd1.out 20.326171875 disk.drbd1285.in0 disk.drbd1584.out0 disk.drbd1356.out0 disk.sda2.in0 disk.drbd1673.out0$ disk./.usageGB578.3581466674805 disk.drbd1289.out0 disk.sda1.in0 disk.drbd1226.out0 disk.drbd1356.in0' proc.memory.availableMB 38503.71875 disk.sda1.out0 disk.drbd2078.in0 disk.drbd1931.out0 disk.drbd2078.out0 disk.drbd1673.in0 disk.drbd1828.out0 disk.drbd1550.out0* proc.memory.percent17.182016372680664 network.sent 1.041283e+07 disk.drbd1828.in0 disk.drbd1289.in0 disk.sda.out 137.23828125 disk.drbd1965.in0 disk.drbd1938.out0xǍ(: ڡU gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts96.081' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent32.027 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0_2R(:  һU network.recv613061 disk.drbd1285.out0 disk.drbd1510.out0 disk.dm-0.in 4.64453125 disk.drbd1226.out0 disk.drbd1931.out0 disk.nvme0n1.in 4.64453125 disk.drbd1938.out0( proc.memory.availableMB 38503.703125* disk./.usagePercent32.344806491655454 disk.sda.in0 proc.cpu.threads29 disk.sda.out 137.64453125 disk.drbd1226.in0 disk.drbd1828.in0 disk.drbd1550.in0 disk.drbd1938.in0 disk.dm-0.out 249.1796875 disk.rbd1.in0 disk.drbd2041.in0 disk.drbd2041.out0 network.sent 1.0457897e+07 disk.drbd1584.in0 disk.drbd1584.out0 disk.drbd1965.out0 disk.nvme0n1.out 249.1796875 disk.drbd1289.in0 disk.drbd1289.out0 disk.drbd1285.in0 disk.sda2.out 137.64453125 disk.drbd1552.in0 disk.drbd1552.out0! proc.memory.rssMB 11260.40625 cpu13.77783835427197$ disk./.usageGB578.3579902648926 disk.drbd1356.in0 disk.rbd0.out0 disk.drbd1510.in0 disk.drbd2118.in0 disk.drbd1817.out0 disk.drbd1550.out0 disk.drbd1828.out0 disk.sda1.in0 disk.drbd1356.out0 disk.drbd1817.in0 disk.drbd1965.in0* proc.memory.percent17.182016372680664 disk.sda1.out0 disk.drbd2118.out0 disk.drbd1673.in0 disk.sda2.in0 disk.drbd2111.in0 disk.drbd2111.out0 disk.rbd1.out 20.326171875 disk.drbd1931.in0 disk.drbd2078.in0 disk.drbd2078.out0 disk.drbd1673.out0 disk.rbd0.in0$ memory_percent41.248011589050292- _wandbruntime5059  _runtime5059< & lr-AdamW/pg10.0002937771537460042 & lr-AdamW/pg20.0002937771537460042 trainer/global_step11399 % _timestamp1.7858003464538157e+09 _runtime5059.276786224 _step908v & lr-AdamW/pg10.0002937771537460042 & lr-AdamW/pg20.0002937771537460042 trainer/global_step11399 % _timestamp1.7858003464538157e+09 _runtime5059.276786224 _step908qB2- _wandbruntime5059  _runtime5059'O `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed10301941783040 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed10030715228672 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10030715228672 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated190842526164832 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10030802658304 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed77546199 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated205736443629568 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed205736339160576 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated32662272 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 120390761 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 120390761 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 120381105 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 196125540 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed44884195 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed190842509125472 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed32662004 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed196166955854848 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated195705640971264 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed195705623931904 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed206468897637888 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated10030802658304 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 _step909 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated75734779 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current268 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548Em'O @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10030802658304 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed77546199 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated205736443629568 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed205736339160576 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated32662272 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 120390761 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 120390761 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 120381105 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 196125540 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed44884195 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed190842509125472 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed32662004 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed196166955854848 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated195705640971264 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed195705623931904 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed206468897637888 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated10030802658304 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 _step909 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated75734779 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current268 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed10301941783040 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed10030715228672 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10030715228672 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated190842526164832 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448R(2- _wandbruntime5059  _runtime5059r|M trainer/global_step11399 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed195722790694912 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 120391628 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated10031682285056 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10031594848256 QCߚm!DeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated200866396169854 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 196142720 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated10031682285056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed206487005197312 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 120391628 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 _runtime5059.648626269 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated196184187875328 % _timestamp1.7858003468252437e+09 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed75741420 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated205754490019328 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed77552982 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated44888134 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated75741422 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated205754490019328 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 120401298 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated206487169156608 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed10031594848256 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated32665078 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 196142720 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed196184162971648 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated195722807734272 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed195722790694912 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated195722807734272 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current228 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed205754385543168 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated190859266337152 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 196133048 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed75741420 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated10007129832702 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936{&M NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated75741422 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated205754490019328 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 120401298 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated206487169156608 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed10031594848256 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated32665078 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 196142720 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed196184162971648 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated195722807734272 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed195722790694912 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated195722807734272 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current228 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed205754385543168 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated190859266337152 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 196133048 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed75741420 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated10007129832702 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 trainer/global_step11399 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed195722790694912 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 120391628 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated10031682285056 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10031594848256 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated200866396169854 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 196142720 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated10031682285056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed206487005197312 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 120391628 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 _runtime5059.648626269 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated196184187875328 % _timestamp1.7858003468252437e+09 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed75741420 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated205754490019328 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed77552982 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated44888134 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0Ddv2- _wandbruntime5059  _runtime50596Dh $ _timestamp1.785800346826907e+09 /loss/FBt1/detection_loss1.1456020414435233 2loss/SRqqt2/assignment_loss0.9386392829359286 1loss/SRqqt2/detection_loss1.0550694152922697 _runtime5059.649748684 0loss/FRt1/assignment_loss2.6374012964467206 /loss/FBt2/detection_loss1.2881857852141063 trainer/global_step11399 _step911 &loss/total_loss17.085530704554046 epoch0 /loss/FRt2/assignment_loss2.435974835604429 2loss/SRqqt1/assignment_loss0.9257631388873878 3loss/FBt1/assignment_loss5.960459667711313e-08 0loss/FBt2/assignment_loss0.0740579852717076 /loss/FRt2/detection_loss2.4100018938072028 1loss/SRqqt1/detection_loss1.7297684556211939 /loss/FRt1/detection_loss2.4450665144249797weC  /loss/FBt1/detection_loss1.1456020414435233 2loss/SRqqt2/assignment_loss0.9386392829359286 1loss/SRqqt2/detection_loss1.0550694152922697 _runtime5059.649748684 0loss/FRt1/assignment_loss2.6374012964467206 /loss/FBt2/detection_loss1.2881857852141063 trainer/global_step11399 _step911 &loss/total_loss17.085530704554046 epoch0 /loss/FRt2/assignment_loss2.435974835604429 2loss/SRqqt1/assignment_loss0.9257631388873878 3loss/FBt1/assignment_loss5.960459667711313e-08 0loss/FBt2/assignment_loss0.0740579852717076 /loss/FRt2/detection_loss2.4100018938072028 1loss/SRqqt1/detection_loss1.7297684556211939 /loss/FRt1/detection_loss2.4450665144249797 $ _timestamp1.785800346826907e+09\-$(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts108.395' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent36.13166666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0S(:  U disk.sda2.out 138.1171875 disk.drbd1817.out0 disk.drbd1828.in0 disk.drbd2118.out0 disk.drbd1356.in0 disk.drbd1931.out0 disk.drbd1550.in0 disk.sda.in0 cpu13.777222729788507 disk.drbd2078.in0 disk.drbd1931.in0 disk.drbd1285.in0( proc.memory.availableMB 38503.921875 disk.drbd2111.out0 disk.sda1.in0 disk.dm-0.out 249.41796875 disk.drbd1584.out0 disk.rbd1.in0 disk.drbd1552.out0 disk.drbd1289.out0 disk.rbd0.in0 disk.drbd1510.out0 disk.drbd1226.out0 disk.drbd1965.out0 disk.drbd1938.in0 disk.rbd0.out0* proc.memory.percent17.182016372680664$ disk./.usageGB578.3580703735352 disk.drbd1828.out0$ memory_percent41.24767780303955 disk.drbd1817.in0 disk.drbd1510.in0 disk.drbd1584.in0 disk.drbd1356.out0 disk.drbd1550.out0 disk.drbd1938.out0 network.recv614156 disk.nvme0n1.in 4.64453125 disk.drbd2041.in0 proc.cpu.threads29 disk.drbd2118.in0 disk.sda2.in0 disk.rbd1.out 20.3583984375 disk.drbd1965.in0 disk.drbd2041.out0 disk.drbd1226.in0 disk.drbd2078.out0! disk.nvme0n1.out 249.41796875) disk./.usagePercent32.34481097175011 disk.drbd2111.in0 disk.drbd1552.in0 disk.drbd1289.in0 disk.sda1.out0 disk.sda.out 138.1171875! proc.memory.rssMB 11260.40625 disk.drbd1673.in0 disk.drbd1673.out0 disk.drbd1285.out0 network.sent 1.0460944e+07 disk.dm-0.in 4.64453125s'2- _wandbruntime5081  _runtime5081< _step912 & lr-AdamW/pg10.0002950657630702695 & lr-AdamW/pg20.0002950657630702695 trainer/global_step11449 % _timestamp1.7858003686716835e+09 _runtime5081.494638244! & lr-AdamW/pg10.0002950657630702695 & lr-AdamW/pg20.0002950657630702695 trainer/global_step11449 % _timestamp1.7858003686716835e+09 _runtime5081.494638244 _step912-[C2- _wandbruntime5081  _runtime5081B#'O N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 196985692 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated197027336598528 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed206638673446400 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 120909107 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated76066929 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10074798793728 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed10050055019752 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10074711364096 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated191679534780832 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated196563979121664 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed10074711364096 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed76066927 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated10074798793728 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated206638777915392 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed206638673446400 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed196563962082304 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed201729572761224 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed191679517741472 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed197027311694848 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 196976034 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated206638777915392 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed77887002 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated32806206 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 $ _timestamp1.785800368672608e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated45081047 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed45081045 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _step913 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed207374394364928 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 _runtime5081.495900787 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 120918763 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated10050140822876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed32805957 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated201729675603708 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated207374549942784 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 trainer/global_step11449 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated196563979121664 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current251 Bѻ_train_batch_start/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak1308'O F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated76066929 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 120909107 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 196985692 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated197027336598528 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed206638673446400 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 120909107 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated76066929 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10074798793728 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed10050055019752 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10074711364096 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated191679534780832 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated196563979121664 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed10074711364096 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed76066927 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated10074798793728 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated206638777915392 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed206638673446400 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed196563962082304 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed201729572761224 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed191679517741472 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed197027311694848 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 196976034 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated206638777915392 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed77887002 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated32806206 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 $ _timestamp1.785800368672608e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated45081047 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed45081045 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _step913 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed207374394364928 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 _runtime5081.495900787 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 120918763 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated10050140822876 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed32805957 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated201729675603708 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated207374549942784 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 trainer/global_step11449 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated196563979121664 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current251 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 197002872 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 196993200 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed201747190406766 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated76073572 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated201747293249362 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 196993200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated206656824305152 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated197044543715328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed191696257913792 % _timestamp1.7858003690474632e+09 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated196581145884672 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated77894081 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current256 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed197044518811648 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 197002872 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated191696274953152 trainer/global_step11449 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated76073572 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed76073570 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed196581128845312 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 120929300 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 120919630 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed32808841 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10347982440960 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _step914 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed77893823 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated32809097 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 120929300 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated10075678420480 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated196581145884672 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10075590983680 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 120919630W&M [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated196581145884672 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated77894081 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current256 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed197044518811648 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 197002872 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated191696274953152 trainer/global_step11449 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated76073572 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed76073570 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed196581128845312 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 120929300 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 120919630 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed32808841 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10347982440960 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _step914 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed77893823 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated32809097 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 120929300 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated10075678420480 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated196581145884672 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10075590983680 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 120919630 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated10348119399424 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated10075678420480 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 197002872 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 196993200 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed201747190406766 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated76073572 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated201747293249362 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 196993200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated206656824305152 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated197044543715328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed191696257913792 % _timestamp1.7858003690474632e+09ŵqT2- _wandbruntime5081  _runtime5081 0loss/FRt1/assignment_loss2.7612444225110506 /loss/FRt2/assignment_loss2.178281823514213 %loss/total_loss16.60880151497455 trainer/global_step11449 _runtime5081.872082642 /loss/FRt1/detection_loss2.4745891376545557 .loss/FRt2/detection_loss2.616305439824789 epoch0 $ _timestamp1.785800369049206e+09 _step915 1loss/SRqqt1/assignment_loss1.052237964514643 3loss/FBt1/assignment_loss4.006744713791477e-07 4loss/FBt2/assignment_loss0.00019588106428154319 1loss/SRqqt1/detection_loss1.5489683716134592 2loss/SRqqt2/assignment_loss0.8041557073426832 1loss/SRqqt2/detection_loss0.9675233694301411 /loss/FBt1/detection_loss0.6262499448170679 .loss/FBt2/detection_loss1.579049052013194R  0loss/FRt1/assignment_loss2.7612444225110506 /loss/FRt2/assignment_loss2.178281823514213 %loss/total_loss16.60880151497455 trainer/global_step11449 _runtime5081.872082642 /loss/FRt1/detection_loss2.4745891376545557 .loss/FRt2/detection_loss2.616305439824789 epoch0 $ _timestamp1.785800369049206e+09 _step915 1loss/SRqqt1/assignment_loss1.052237964514643 3loss/FBt1/assignment_loss4.006744713791477e-07 4loss/FBt2/assignment_loss0.00019588106428154319 1loss/SRqqt1/detection_loss1.5489683716134592 2loss/SRqqt2/assignment_loss0.8041557073426832 1loss/SRqqt2/detection_loss0.9675233694301411 /loss/FBt1/detection_loss0.6262499448170679 .loss/FBt2/detection_loss1.579049052013194 (:  ÏV disk.drbd1550.out0 disk.drbd1289.out0 disk.drbd1938.in0 disk.drbd1510.out0$ memory_percent41.24698042869568 disk.sda2.out 138.2890625 disk.rbd0.in0 disk.dm-0.in 4.64453125 disk.drbd1817.in0 disk.drbd2118.in0 disk.drbd1289.in0 disk.drbd1938.out0 disk.sda.out 138.2890625* disk./.usagePercent32.344812038439315 disk.rbd1.in0 disk.drbd1931.out0 disk.rbd0.out0 disk.drbd1584.out0 disk.nvme0n1.in 4.64453125$ disk.nvme0n1.out250.26123046875 disk.drbd1673.in0 disk.sda1.out0 disk.drbd1285.out0 disk.drbd1226.out0 disk.sda2.in0 disk.drbd1673.out0 disk.dOrbd2041.out0 disk.drbd1552.out0 network.recv615866 disk.drbd1817.out0 disk.drbd1552.in0 disk.drbd1356.in0 disk.drbd1356.out0 disk.drbd1550.in0 disk.drbd1510.in0! disk.dm-0.out250.26123046875 disk.drbd2111.in0 disk.drbd1965.in0 cpu13.77651060704172 disk.drbd1931.in0 disk.drbd2078.out0 network.sent 1.0506032e+07 disk.drbd2111.out0 disk.drbd1285.in0! proc.memory.rssMB 11260.40625* proc.memory.percent17.182016372680664 proc.cpu.threads29 disk.drbd2078.in0 disk.drbd2041.in0 disk.drbd1828.in0 disk.drbd1828.out0* proc.memory.availableMB38504.37890625 disk.sda.in0$ disk./.usageGB578.3580894470215 disk.drbd1584.in0 disk.drbd2118.out0 disk.drbd1965.out0 disk.rbd1.out 20.3583984375 disk.sda1.in0 disk.drbd1226.in004(: V gpu.0.gpu41.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts102.977' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent34.32566666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0󙐙(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts96.44' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent32.14666666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0XY(:  V disk.sda1.out0 disk.drbd1510.out0 disk.rbd1.in0 disk.drbd1965.out0 disk.drbd1584.out0" disk.nvme0n1.out 250.763671875! proc.memory.rssMB 11260.40625 disk.drbd1673.out0 disk.drbd1550.out0 disk.drbd1289.in0 disk.drbd1285.in0 disk.drbd1226.in0 disk.drbd1356.out0 network.sent 1.0551104e+07 disk.sda2.out 138.76171875 cpu13.775974283074783 disk.drbd1673.in0 disk.drbd1931.in0 disk.drbd1550.in0 disk.dm-0.in 4.64453125 disk.nvme0n1.in 4.64453125$ disk./.usageGB578.3572273254395 disk.drbd1828.in0 disk.drbd1510.in0 disk.drbd1817.in0$ memory_percent41.24816060066223* proc.memory.percent17.182016372680664 disk.drbd1931.out0 disk.drbd1828.out0 disk.drbd1289.out0 disk.drbd1938.in0 disk.rbd0.in0 disk.drbd2111.in0 disk.rbd0.out0 disk.sda.in0 disk.dm-0.out 250.763671875 disk.drbd1965.in0 disk.drbd1285.out0 disk.drbd2118.out0 disk.drbd1226.out0 disk.drbd1584.in0 disk.sda2.in0 disk.drbd1817.out0) disk./.usagePercent32.34476382408728 disk.drbd2041.in0 disk.drbd2078.out0 disk.drbd2118.in0 disk.drbd1552.in0 disk.drbd2111.out0 disk.rbd1.out 20.6005859375 disk.drbd2041.out0 disk.drbd1938.out0 disk.sda1.in0 disk.drbd2078.in0 disk.drbd1356.in0* proc.memory.availableMB38503.60546875 proc.cpu.threads29 network.recv617577 disk.drbd1552.out0 disk.sda.out 138.761718755`2- _wandbruntime5103  _runtime51031> ' lr-AdamW/pg10.00029635437239453483 ' lr-AdamW/pg20.00029635437239453483 trainer/global_step11499 % _timestamp1.7858003905872784e+09 _runtime5103.410282891 _step9169S7 ' lr-AdamW/pg20.00029635437239453483 trainer/global_step11499 % _timestamp1.7858003905872784e+09 _runtime5103.410282891 _step916 ' lr-AdamW/pg10.00029635437239453483)4.^2- _wandbruntime5103  _runtime5103mx%g'O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed10093943483260 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated10094029286384 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated197887692438528 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed197887667534848 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated197422317272064 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _runtime5103.414723616 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 121437109 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed208279975095296 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed10118707499520 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed202610469840732 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10118794929152 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated45277897 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed207541007732224 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 121437109 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed207541007732224 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 trainer/global_step11499 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated197422317272064 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated78228549 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated32950652 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed192516526357472 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed45277895 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10118707499520 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 _step917 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 197845844 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated10392436137472 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated192516543396832 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed76399077 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current244 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed197422300232704 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated76399079 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 121446765 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated207541112201216 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed32950408 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated197887692438528 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed197887667534848 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated197422317272064 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _runtime5103.414723616 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 121437109 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed208279975095296 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed10118707499520 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed202610469840732 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10118794929152 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated45277897 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed207541007732224 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 121437109 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed207541007732224 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 trainer/global_step11499 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated197422317272064 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated78228549 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated32950652 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed192516526357472 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed45277895 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10118707499520 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 _step917 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 197845844 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated10392436137472 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated192516543396832 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed76399077 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current244 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed197422300232704 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated76399079 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 121446765 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated207541112201216 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed32950408 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 JDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed78235102 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 197853352 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 trainer/global_step11499 _step918 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 197863024 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated78235348 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated192533283569152 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed208298083023360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated76405722 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated10393343233024 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 121457302 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 121447632 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 197853352 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 % _timestamp1.7858003909573066e+09 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 197863024 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 121447632 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed32953270 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated76405722 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed197439466995712 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10119587119104 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed202628087486274 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed192533266529792 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed10094820956482 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed197439466995712 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated207559158590976 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed10119587119104 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated202628190328870 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated10119674555904 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed197904874651648 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10393208371712 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated197904899555328 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 121457302 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed207559054114816 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed76405720 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated32953514 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated208298242788352 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 _runtime5103.780719624 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated10094906759718 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0t &M JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed45281832 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 197853352 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 trainer/global_step11499 _step918 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 197863024 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated78235348 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated192533283569152 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed208298083023360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated76405722 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated10393343233024 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 121457302 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 121447632 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 197853352 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 % _timestamp1.7858003909573066e+09 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 197863024 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 121447632 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed32953270 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated76405722 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed197439466995712 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10119587119104 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed202628087486274 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed192533266529792 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed10094820956482 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed197439466995712 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated207559158590976 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed10119587119104 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated202628190328870 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated10119674555904 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed197904874651648 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10393208371712 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated197904899555328 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 121457302 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed207559054114816 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed76405720 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated32953514 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated208298242788352 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 _runtime5103.780719624 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated10094906759718 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed78235102 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 FDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 % _timestamp1.7858004126520696e+09 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed10162703634944 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed193353534973472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated76731229 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated10137917749892 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed198280638383104 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 trainer/global_step11549 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 _runtime5125.478702599 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed198280638383104 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 121974767 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed10137831946768 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed208443342018048 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed45474745 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed208443342018048 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated198748048278528 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated45474747 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 198696338 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current248 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed203491366920240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated193353552012832 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640~_O JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed208443342018048 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated198748048278528 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated45474747 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 198696338 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current248 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed203491366920240 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated193353552012832 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 121974767 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed76731227 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 121965111 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated33093957 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated209185686761984 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated10437638483456 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated198280655422464 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 % _timestamp1.7858004126520696e+09 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed10162703634944 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed193353534973472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated76731229 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated10137917749892 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed198280638383104 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 trainer/global_step11549 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 _runtime5125.478702599 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed198280638383104 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 121974767 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed10137831946768 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed208443342018048 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed45474745 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632X{2- _wandbruntime5125  _runtime5125X&M ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated76737872 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 198713504 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 _runtime5125.849567602 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed78575231 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed208461388400640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current252 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated45478684 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed209203628487168 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 % _timestamp1.7858004130261097e+09 _step922 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated208461492876800 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated198297822185472 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 198723176 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed198297805146112 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10163583254528 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed76737870 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10438397995520 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed45478682 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated209203792446464 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated198297822185472 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated10163670691328 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated10163670691328 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 198723176 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated33096799 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 121985304>8&M J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed10163583254528 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed203508984565782 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 121975634 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated208461492876800 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated198297822185472 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 198723176 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed198297805146112 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10163583254528 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed76737870 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10438397995520 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed45478682 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated209203792446464 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated198297822185472 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated10163670691328 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated10163670691328 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 198723176 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated33096799 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 121985304 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated76737872 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 198713504 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 _runtime5125.849567602 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed78575231 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed208461388400640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current252 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated45478684 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed209203628487168 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 % _timestamp1.7858004130261097e+09 _step922 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 FDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed77063377 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed33236839 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated209345780772864 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 122493113 _step925 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 199556490 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated33237076 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed209345676303872 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated10482833634816 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed204372263999748 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 122493113 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10206787200000 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed199608379214848 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed10482696669184 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed10181720410276 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated199138993572864 % _timestamp1.7858004358237581e+09 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed199138976533504 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed199138976533504 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated77063379 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated204372366842232 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed78908434 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed10206699770368 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current237 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed45671595 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed209345676303872 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 199566148 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated199138993572864)H'O YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated209345780772864 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 122493113 _step925 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 199556490 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated33237076 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed209345676303872 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated10482833634816 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed204372263999748 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 122493113 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10206787200000 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed199608379214848 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed10482696669184 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed10181720410276 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated199138993572864 % _timestamp1.7858004358237581e+09 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed199138976533504 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed199138976533504 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated77063379 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated204372366842232 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed78908434 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed10206699770368 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current237 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed45671595 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed209345676303872 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 199566148 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated199138993572864 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed77063377 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed77063377 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed33236839 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0--2- _wandbruntime5149  _runtime5149R!&M EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 199583328 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 199583328 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated210109347599360 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10483597308416 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated209363827162624 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 122513306 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed45675532 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 122503636 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed204389881645290 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed199156143296512 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current240 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 _step926 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated10182683686734 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed199156143296512 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 199573656 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10207579389952 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed199625586331648 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed10207579389952 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated199625611235328 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated77070022 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 $ _timestamp1.785800436276541e+09 _runtime5149.100113679 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 199573656 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated77070022 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated45675534 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed78915293 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 122513306 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121*TM REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated77070022 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 $ _timestamp1.785800436276541e+09 _runtime5149.100113679 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 199573656 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated77070022 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated45675534 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed78915293 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 122513306 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 199583328 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 199583328 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated210109347599360 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10483597308416 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated209363827162624 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 122513306 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed45675532 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 122503636 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed204389881645290 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed199156143296512 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current240 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 _step926 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated10182683686734 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed199156143296512 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 199573656 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10207579389952 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed199625586331648 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed10207579389952 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated199625611235328 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032%2- _wandbruntime5149  _runtime5149cn .loss/FRt2/detection_loss2.407078748523462 1loss/SRqqt1/detection_loss1.6203820813292016 .loss/FBt2/detection_loss2.317943751745458 1loss/SRqqt1/assignment_loss1.008262293675216 /loss/FBt1/detection_loss0.7396861228837885 epoch0 % _timestamp1.7858004362781882e+09 _runtime5149.101137601 0loss/FRt2/assignment_loss1.9083210084496476 2loss/FBt2/assignment_loss0.004042940147152356 trainer/global_step11599 0loss/FRt1/assignment_loss2.7718598897519864 2loss/FBt1/assignment_loss0.023403207163286293 .loss/FRt1/detection_loss2.753386369661281 1loss/SRqqt2/detection_loss0.9410895515755293 &loss/total_loss17.479135592294675 _step927 2loss/SRqqt2/assignment_loss0.9836796273886683,  0loss/FRt1/assignment_loss2.7718598897519864 2loss/FBt1/assignment_loss0.023403207163286293 .loss/FRt1/detection_loss2.753386369661281 1loss/SRqqt2/detection_loss0.9410895515755293 &loss/total_loss17.479135592294675 _step927 2loss/SRqqt2/assignment_loss0.9836796273886683 .loss/FRt2/detection_loss2.407078748523462 1loss/SRqqt1/detection_loss1.6203820813292016 .loss/FBt2/detection_loss2.317943751745458 1loss/SRqqt1/assignment_loss1.008262293675216 /loss/FBt1/detection_loss0.7396861228837885 epoch0 % _timestamp1.7858004362781882e+09 _runtime5149.101137601 0loss/FRt2/assignment_loss1.9083210084496476 2loss/FBt2/assignment_loss0.004042940147152356 trainer/global_step11599w (: U gpu.0.gpu38.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts126.256' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.08533333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0+Z(:  U) disk./.usagePercent32.34481673187181 disk.drbd1931.in0 disk.drbd2041.out0 disk.dm-0.in 4.64453125 disk.drbd1584.in0 disk.drbd1965.out0 disk.drbd1817.in0 disk.sda.in0 disk.sda2.in0$ disk./.usageGB578.3581733703613 disk.drbd1552.out0 disk.drbd1938.out0 disk.rbd0.out0 network.sent 1.0647271e+07 disk.drbd1828.in0 disk.drbd1828.out0 disk.drbd1285.in0 disk.drbd1226.in0 disk.drbd2118.in0* proc.memory.availableMB38503.48046875 disk.drbd1673.out0 disk.drbd1550.in0 disk.drbd1285.out0 disk.drbd1510.out0 disk.drbd2078.in0 disk.sda1.out0 disk.sda1.in0$ disk.nvme0n1.out252.82861328125 disk.drbd1356.in0 disk.rbd1.out 20.8828125* proc.memory.percent17.182016372680664 cpu13.772974634040107 network.recv622744 disk.rbd0.in0 disk.drbd2118.out0 disk.drbd1673.in0 disk.drbd1931.out0 disk.drbd1938.in0 disk.drbd1584.out0 disk.drbd1965.in0 disk.drbd1510.in0 disk.nvme0n1.in 4.64453125 disk.drbd1356.out0 disk.drbd1289.out0 disk.drbd2078.out0 disk.drbd2111.out0 disk.drbd1552.in0 disk.sda.out 140.46484375 disk.drbd1226.out0 disk.drbd1817.out0 proc.cpu.threads29$ memory_percent41.24835133552551 disk.drbd1289.in0! disk.dm-0.out252.82861328125 disk.sda2.out 140.46484375 disk.rbd1.in0! proc.memory.rssMB 11260.40625 disk.drbd2041.in0 disk.drbd1550.out0 disk.drbd2111.in02- _wandbruntime5171  _runtime5171^J. & lr-AdamW/pg10.0003002202003673307 & lr-AdamW/pg20.0003002202003673307 trainer/global_step11649 % _timestamp1.7858004588391442e+09 _runtime5171.662103838 _step928 _runtime5171.662103838 _step928 & lr-AdamW/pg10.0003002202003673307 & lr-AdamW/pg20.0003002202003673307 trainer/global_step11649 % _timestamp1.7858004588391442e+09DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated199997331723264 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 123021115 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated45868447 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated200468759958528 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed199997314683904 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed199997314683904 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10250695905792 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed77395527 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed205253161079256 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 % _timestamp1.7858004588400822e+09 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated205253263921740 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed210248010589696 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 123021115 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated210248115058688 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed77395527 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed200468735054848 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed210248010589696 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated79248829 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current249 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 _step929 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated77395529 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated10528025001472 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 trainer/global_step11649 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak1218'O ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated205253263921740 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed210248010589696 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 123021115 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated210248115058688 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed77395527 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed200468735054848 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed210248010589696 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated79248829 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current249 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 _step929 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated77395529 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated10528025001472 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 trainer/global_step11649 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated199997331723264 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 123021115 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated45868447 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated200468759958528 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed199997314683904 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed199997314683904 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10250695905792 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed77395527 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed205253161079256 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 % _timestamp1.7858004588400822e+09 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current249036808=k;2- _wandbruntime5172  _runtime5172I { M D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 200433808 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed200014481446912 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 123031638 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed210266056972288 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated77402172 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated200485967075328 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed77402170 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed200485942171648 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 123041308 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated200014498486272 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed45872382 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 126472704 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated211014893652992 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed79255328 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed10226486347006 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated195044309417152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 123041308 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated45872384 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated33383180 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 151376384 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 _runtime5172.040020547 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 200443480 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated210266161448448 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 200443480 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed10251575525376 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed211014742276608pE&M OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current234 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 $ _timestamp1.785800459216599e+09 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 _step930 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed205270778724798 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed195044292377792 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated195044309417152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 123041308 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated45872384 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated33383180 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 151376384 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 _runtime5172.040020547 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 200443480 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated210266161448448 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 200443480 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed10251575525376 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed211014742276608 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 200433808 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed200014481446912 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 123031638 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed210266056972288 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated77402172 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated200485967075328 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed77402170 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed200485942171648 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 123041308 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated200014498486272 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed45872382 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 126472704 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated211014893652992 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed79255328 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed10226486347006 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 EDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated211150449344512 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed211150344875520 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10294692041216 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated46065297 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated33524226 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed77727677 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 trainer/global_step11699 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 201276794 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed77727677 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed46065295 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 123558773 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated10573244257280 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 123549117 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated200855669873664 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated206134161001248 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated200855669873664 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed10573113583104 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current261 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 _runtime5193.775951483 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed200855652834304 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 123558773 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed201329090894848 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 # _timestamp1.78580048095266e+09 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed10269497337292 QBDeviceStatsMonitor.on_train_batch_start/resGW4erved_bytes.all.current 3021996032 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251ם'O G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed46065295 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 123558773 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated10573244257280 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 123549117 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated200855669873664 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated206134161001248 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated200855669873664 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed10573113583104 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current261 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 _runtime5193.775951483 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed200855652834304 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 123558773 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed201329090894848 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 # _timestamp1.78580048095266e+09 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed10269497337292 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed10294692041216 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed211902204477952 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10294779470848 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated211150449344512 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed211150344875520 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10294692041216 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated46065297 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated33524226 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed77727677 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 trainer/global_step11699 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 201276794 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed77727677 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0؃2- _wandbruntime5194  _runtime5194K< &M M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 201303632 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current247 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated206151778646902 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed206151675804306 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated33527037 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed195881300993792 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed200872819597312 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 123569310 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 trainer/global_step11699 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 _runtime5194.146130261 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed77734320 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed79596024 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed46069232 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed77734320 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated200872836636672 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated201346322915328 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current245 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed10295571660800 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed211168391258112 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated46069234 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed33526792 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 201293960 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated10295659097600 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed211920310194176 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated77734322 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 123559640 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed201346298011648 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 201293960 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 % _timestamp1.7858004813227232e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated10574144946688 _step934!&M UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed77734320 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated200872836636672 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated201346322915328 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current245 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed10295571660800 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed211168391258112 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated46069234 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed33526792 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 201293960 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated10295659097600 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed211920310194176 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated77734322 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 123559640 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed201346298011648 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 201293960 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 % _timestamp1.7858004813227232e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated10574144946688 _step934 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 201303632 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current247 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated206151778646902 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed206151675804306 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated33527037 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed195881300993792 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed200872819597312 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 123569310 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 trainer/global_step11699 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 _runtime5194.146130261 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed77734320 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed79596024 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed46069232 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888I2- _wandbruntime5194  _runtime5194I)  2loss/SRqqt1/assignment_loss0.7195954898165332 /loss/FBt2/detection_loss0.8510599056155317 _runtime5194.147371849 0loss/FRt1/assignment_loss2.7233263716885916 1loss/SRqqt2/detection_loss0.9893873270880431 %loss/total_loss15.97483558309111 epoch0 % _timestamp1.7858004813245041e+09 _step935 4loss/FBt2/assignment_loss1.4371156188038488e-06 /loss/FRt1/detection_loss2.6805664771481563 /loss/FRt2/detection_loss2.5215577373038167 1loss/SRqqt1/detection_loss1.7571601431506376 /loss/FBt1/detection_loss0.9128372068093582 trainer/global_step11699 0loss/FRt2/assignment_loss1.8332465515620466 2loss/SRqqt2/assignment_loss0.9860874754578496 3loss/FBt1/assignment_loss9.460334927589222e-06{E 3loss/FBt1/assignment_loss9.460334927589222e-06 2loss/SRqqt1/assignment_loss0.7195954898165332 /loss/FBt2/detection_loss0.8510599056155317 _runtime5194.147371849 0loss/FRt1/assignment_loss2.7233263716885916 1loss/SRqqt2/detection_loss0.9893873270880431 %loss/total_loss15.97483558309111 epoch0 % _timestamp1.7858004813245041e+09 _step935 4loss/FBt2/assignment_loss1.4371156188038488e-06 /loss/FRt1/detection_loss2.6805664771481563 /loss/FRt2/detection_loss2.5215577373038167 1loss/SRqqt1/detection_loss1.7571601431506376 /loss/FBt1/detection_loss0.9128372068093582 trainer/global_step11699 0loss/FRt2/assignment_loss1.8332465515620466 2loss/SRqqt2/assignment_loss0.98608747545784961+(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts126.192' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.06399999999999 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0ۼ2(:  U disk.rbd1.out 20.9150390625 disk.drbd1289.in0 disk.sda1.in0 disk.drbd2111.in0) disk./.usagePercent32.34470963627571 disk.drbd1931.in0 disk.drbd1510.in0 disk.drbd1510.out0 disk.drbd1226.out0 disk.drbd2078.out0# disk.nvme0n0*(1.out253.7451171875 disk.drbd1965.in0 disk.dm-0.in 4.64453125 disk.drbd1584.in0 disk.drbd1673.out0 disk.drbd1552.in0 disk.drbd1828.out0 disk.sda.out 141.43359375 disk.drbd2111.out0 disk.drbd2118.out0 disk.drbd1356.out0 disk.drbd1828.in0! proc.memory.rssMB 11260.40625 disk.drbd1965.out0 disk.drbd2041.out0 disk.drbd1285.out0# disk./.usageGB578.356258392334 disk.drbd1285.in0 disk.drbd1226.in0 disk.drbd2078.in0 disk.drbd1673.in0 disk.drbd1550.out0 disk.drbd1289.out0 disk.sda2.in0 proc.cpu.threads29 disk.drbd1356.in0 disk.drbd1931.out0 disk.drbd1552.out0 disk.sda1.out0$ memory_percent41.24811291694641 disk.nvme0n1.in 4.64453125 disk.drbd1817.out0 disk.rbd1.in0* proc.memory.percent17.182016372680664 disk.drbd1550.in0 cpu13.771070556896891 network.recv627383 disk.drbd1938.out0 disk.sda.in0 disk.dm-0.out253.7451171875 network.sent 1.0740749e+07 disk.sda2.out 141.43359375 disk.drbd2041.in0 disk.rbd0.in0 disk.drbd2118.in0 disk.drbd1938.in0 disk.rbd0.out0 disk.drbd1584.out0* proc.memory.availableMB38503.63671875 disk.drbd1817.in02- _wandbruntime5215  _runtime5215 _runtime5215.695017804 _step936 ' lr-AdamW/pg10.00030279741901586134 ' lr-AdamW/pg20.00030279741901586134 trainer/global_step11749 % _timestamp1.7858005028720589e+09Vd ' lr-AdamW/pg10.00030279741901586134 ' lr-AdamW/pg20.00030279741901586134 trainer/global_step11749 % _timestamp1.7858005028720589e+09 _runtime5215.695017804 _step936A2- _wandbruntime5215  _runtime52152c{'O RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 % _timestamp1.7858005028729966e+09 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated10618461231616 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated10313471603924 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated201714008024064 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 _step937 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 _runtime5215.696436493 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated78059829 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed212807775195136 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated202189471638528 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated212052783630336 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated212807932870144 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated33667561 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 124077119 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 202146604 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current235 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated212052783630336 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed79929473 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed10338688176640 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed202189446734848 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed212052679161344 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed10618328460288 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated207015058080756 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated78059829 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed212052679161344 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated202189471638528 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated212052783630336 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated212807932870144 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated33667561 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 124077119 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 202146604 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current235 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated212052783630336 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed79929473 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed10338688176640 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed202189446734848 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed212052679161344 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed10618328460288 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated207015058080756 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated78059829 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed212052679161344 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 J2- _wandbruntime5216  _runtime5216"A&M T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated212070830020096 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated201731174787072 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 166056448 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated33670322 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed78066470 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed10339567796224 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 202154112 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed79936181 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 124097312 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed10314263274022 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated10339655233024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated10314349077258 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed78066470 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 _runtime 5216.08204851 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated78066472 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 124097312 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated202206678755328 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10619224798720 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed201731157747712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed46266082 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 % _timestamp1.7858005032586637e+09 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 _step938 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated212070830020096 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 202163784 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated46266084 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated10619365951488 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated196718326649152 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed212825878650368 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated10339655233024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated78066472 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed207032572883814 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0jQM SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 124097312 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 XJDeviceStatsMonitor.on_train_bam*5%tch_end/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed10314263274022 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated10339655233024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated10314349077258 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed78066470 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 _runtime 5216.08204851 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated78066472 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 124097312 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated202206678755328 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10619224798720 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed201731157747712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed46266082 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 % _timestamp1.7858005032586637e+09 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 _step938 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated212070830020096 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 202163784 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated46266084 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated10619365951488 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated196718326649152 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed212825878650368 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated10339655233024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated78066472 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed207032572883814 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated212070830020096 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated201731174787072 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 166056448 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated33670322 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed78066470 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed10339567796224 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 202154112 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed79936181 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections05/2- _wandbruntime5216  _runtime5216rLL  0loss/FRt1/assignment_loss2.7341630499014693 /loss/FBt1/detection_loss0.9468477125046775 /loss/FBt2/detection_loss1.9220190849155188 trainer/global_step11749 _step939 0loss/FRt2/assignment_loss1.7931275339353652 2loss/SRqqt1/assignment_loss0.9025610894983951 4loss/FBt2/assignment_loss0.00019632672123037764 .loss/FRt1/detection_loss2.557297110557556 /loss/FRt2/detection_loss2.1157262380279245 1loss/SRqqt1/detection_loss1.6684834730793845 1loss/SRqqt2/detection_loss1.0219072562429201 &loss/total_loss16.495845317487817 _runtime 5216.083186 2loss/SRqqt2/assignment_loss0.8334796293659685 3loss/FBt1/assignment_loss3.681273740774316e-05 epoch0 % _timestamp1.7858005032603576e+09 0loss/FRt1/assignment_loss2.7341630499014693 /loss/FBt1/detection_loss0.9468477125046775 /loss/FBt2/detection_loss1.9220190849155188 trainer/global_step11749 _step939 0loss/FRt2/assignment_loss1.7931275339353652 2loss/SRqqt1/assignment_loss0.9025610894983951 4loss/FBt2/assignment_loss0.00019632672123037764 .loss/FRt1/detection_loss2.557297110557556 /loss/FRt2/detection_loss2.1157262380279245 1loss/SRqqt1/detection_loss1.6684834730793845 1loss/SRqqt2/detection_loss1.0219072562429201 &loss/total_loss16.495845317487817 _runtime 5216.083186 2loss/SRqqt2/assignment_loss0.8334796293659685 3loss/FBt1/assignment_loss3.681273740774316e-05 epoch0 % _timestamp1.7858005032603576e+09 k(: U gpu.0.gpu28.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts127.303' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent42.434333333333335 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0`}U(:  U disk.drbd1552.out0 disk.drbd1285.out0 disk.rbd0.out0 disk.dm-0.in 4.64453125 disk.drbd1356.in0 disk.drbd2118.in0 disk.nvme0n1.in 4.64453125% memory_percent41.248250007629395 disk.drbd2041.in0 disk.drbd1289.out0 disk.sda.in0 disk.drbd2078.in0 disk.drbd1584.in0# disk.nvme0n1.out253.9794921875 disk.drbd1289.in0 disk.drbd2118.out0 cpu13.770728052240324 network.sent 1.078578e+07 disk.drbd1673.in0 disk.drbd1550.in0 disk.rbd0.in0 network.recv629155$ disk./.usageGB578.3562774658203 disk.drbd1931.in0 disk.drbd1817.in0 disk.sda1.out0 disk.drbd1285.in0 disk.drbd1226.out0 disk.sda2.in0 disk.drbd1965.in0 disk.drbd1552.in0 disk.drbd1828.in0 disk.drbd1828.out0 disk.drbd1938.in0 disk.drbd1938.out0 disk.sda2.out 141.890625 disk.drbd2041.out0 disk.drbd2078.out0 proc.cpu.threads29 disk.drbd2111.in0 disk.drbd1673.out0 disk.drbd1931.out0 disk.drbd1510.in0 disk.drbd1584.out0! proc.memory.rssMB 11260.40625 disk.drbd2111.out0 disk.drbd1550.out0 disk.sda1.in0 disk.drbd1510.out0 disk.rbd1.in0 disk.rbd1.out 21.1572265625 disk.dm-0.out253.9794921875 disk.drbd1226.in0 disk.drbd1356.out0( proc.memory.availableMB 38503.546875* proc.memory.percent17.182016372680664) disk./.usagePercent32.34471070296491 disk.drbd1817.out0 disk.drbd1965.out0 disk.sda.out 141.890625s82(: U gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts97.326' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent32.442 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0 0\(:  V disk.drbd1817.in0 disk.drbd1550.out0 disk.drbd2078.in0$ memory_percent41.24862551689148 disk.drbd2078.out0 disk.drbd1584.out0 disk.drbd1817.out0 disk.drbd1828.out0 disk.sda1.out0 disk.drbd1356.in0 disk.drbd2111.in0 disk.drbd1965.in0 disk.drbd1673.out0 disk.drbd1285.in0 disk.drbd1226.out0 disk.nvme0n1.in 4.64453125$ disk./.usageGB578.3562202453613 disk.drbd2111.out0 disk.drbd1931.in0 disk.drbd1584.in0$ disk.nvme0n1.out254.26806640625* proc.memory.availableMB38503.30078125 network.sent 1.083079e+07 disk.drbd1552.out0 disk.drbd1289.in0 disk.drbd1289.out0 disk.sda2.out 142.34765625 disk.drbd1965.out0 disk.drbd1931.out0 disk.rbd0.out0 disk.drbd1510.in0 disk.drbd1226.in0 proc.cpu.threads29 disk.drbd1356.out0 disk.sda2.in0 disk.rbd1.in0 disk.drbd1828.in0 disk.drbd1285.out0 disk.drbd2041.out0 disk.sda.in0 disk.sda1.in0 disk.dm-0.in 4.64453125! disk.dm-0.out254.26806640625! proc.memory.rssMB 11260.40625* proc.memory.percent17.182016372680664 cpu13.770149343360263 disk.drbd1552.in0 network.recv630656 disk.rbd1.out 21.1572265625 disk.rbd0.in0 disk.sda.out 142.34765625 disk.drbd2118.out0) disk./.usagePercent32.34470750289729 disk.drbd1550.in0 disk.drbd2118.in0 disk.drbd2041.in0 disk.drbd1938.out0 disk.drbd1673.in0 disk.drbd1938.in0 disk.drbd1510.out0Q:2- _wandbruntime5238  _runtime52384 _runtime 5238.09554655 _step940 ' lr-AdamW/pg10.00030408602834012665 ' lr-AdamW/pg20.00030408602834012665 trainer/global_step11799 # _timestamp1.78580052527256e+09Św ' lr-AdamW/pg20.00030408602834012665 trainer/global_step11799 # _timestamp1.78580052527256e+09 _runtime 5238.09554655 _step940 ' lr-AdamW/pg10.00030408602834012665&vc2- _wandbruntime5238  _runtime5238v'O UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed212955013447168 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated33809991 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated78391979 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated203049827478528 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed197538578053472 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated46458997 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated212955117916160 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 202997098 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 124614777 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed207895852317780 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed78391977 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed212955013447168 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed203049802574848 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated197538595092832 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 124605121 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated202572346174464 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated10357360067432 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10382684312064 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 124605121 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated10382771741696 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10382771741696 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 124382720 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated78391979 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed80268735 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed33809740 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated203049827478528 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed197538578053472 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated46458997 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated212955117916160 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 202997098 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 124614777 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed207895852317780 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed78391977 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed212955013447168 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed203049802574848 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated197538595092832 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 124605121 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated202572346174464 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated10357360067432 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10382684312064 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 124605121 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated10382771741696 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10382771741696 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 124382720 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated78391979 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed80268735 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed33809740 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10664439305216 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed202589495898112 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 124615644 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated213731606664704 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed207913469963322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed10358151737530 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated212973164305920 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 203023936 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed202589495898112 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 trainer/global_step11799 $ _timestamp1.785800525657428e+09 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 124625314 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated80275869 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed80275608 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated202589512937472 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 124615644 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated46462934 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated10383651368448 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current259 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime5238.480840398 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed197555318225792 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed46462932 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 203023936 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 _step942 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated207913572805918 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 203014264 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10383563931648 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed212973059829760 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 203023936 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 _step942 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated207913572805918 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 203014264 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10383563931648 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed212973059829760 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10664439305216 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed202589495898112 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 124615644 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated213731606664704 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed207913469963322 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed10358151737530 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated212973164305920 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 203023936 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed202589495898112 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 trainer/global_step11799 $ _timestamp1.785800525657428e+09 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 124625314 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated80275869 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed80275608 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated202589512937472 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 124615644 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated46462934 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated10383651368448 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current259 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime5238.480840398 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed197555318225792 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed46462932lqu2- _wandbruntime5238  _runtime52388  0loss/FRt2/assignment_loss1.8768079304695129 3loss/FBt2/assignment_loss4.999400462111225e-05 /loss/FRt2/detection_loss2.3369656935334207 /loss/FRt1/detection_loss2.3815703276130886 1loss/SRqqt1/detection_loss1.7798553690010188 &loss/total_loss15.671199980936676 _runtime 5238.4825441 _step943 1loss/SRqqt1/assignment_loss0.783441590430329 2loss/SRqqt2/assignment_loss0.8308617409335135 3loss/FBt1/assignment_loss0.0009194908370058065 1loss/SRqqt2/detection_loss0.7811145491736091 /loss/FBt1/detection_loss1.0075894246899313 /loss/FBt2/detection_loss1.4896294655743987 trainer/global_step11799 $ _timestamp1.785800525659193e+09 0loss/FRt1/assignment_loss2.4023944046762256 epoch0urZ 0loss/FRt2/assignment_loss1.8768079304695129 3loss/FBt2/assignment_loss4.999400462111225e-05 /loss/FRt2/detection_loss2.3369656935334207 /loss/FRt1/detection_loss2.3815703276130886 1loss/SRqqt1/detection_loss1.7798553690010188 &loss/total_loss15.671199980936676 _runtime 5238.4825441 _step943 1loss/SRqqt1/assignment_loss0.783441590430329 2loss/SRqqt2/assignment_loss0.8308617409335135 3loss/FBt1/assignment_loss0.0009194908370058065 1loss/SRqqt2/detection_loss0.7811145491736091 /loss/FBt1/detection_loss1.0075894246899313 /loss/FBt2/detection_loss1.4896294655743987 trainer/global_step11799 $ _timestamp1.785800525659193e+09 0loss/FRt1/assignment_loss2.4023944046762256 epoch0(: үU gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts113.624' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent37.87466666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0pT(:  U) disk./.usagePercent32.34471070296491 disk.drbd2041.in0 disk.sda.out142.5625 disk.drbd1584.out0 disk.drbd2118.out0 disk.rbd1.in0 network.recv631686 disk.drbd1931.in0 disk.drbd1289.out0 disk.sda1.in0* proc.memory.percent17.182016372680664 disk.sda.in0 disk.drbd2111.in0 disk.drbd2111.out0 proc.cpu.threads29 disk.drbd1285.in0 disk.drbd1226.out0 disk.drbd1965.in0 disk.drbd1285.out0 disk.drbd1828.out0! disk.dm-0.out254.51025390625 disk.sda1.out0$ disk./.usageGB578.3562774658203 disk.drbd1938.in0 disk.rbd0.out0 cpu13.769551880808152 network.sent 1.0833805e+07 disk.drbd1673.out0 disk.drbd1289.in0 disk.drbd1817.in0 disk.drbd1673.in0 disk.drbd1510.in0 disk.drbd1226.in0 disk.drbd2118.in0 disk.drbd1356.in0 disk.drbd1356.out0 disk.sda2.out142.5625 disk.drbd1931.out0 disk.drbd1552.out0 disk.rbd0.in0 disk.drbd1828.in0* proc.memory.availableMB38503.51953125 disk.dm-0.in 4.64453125 disk.drbd2078.out0 disk.drbd1584.in0 disk.rbd1.out 21.189453125! proc.memory.rssMB 11260.40625 disk.drbd1550.out0 disk.drbd2078.in0$ memory_percent41.24829173088074 disk.drbd1552.in0 disk.drbd1510.out0 disk.nvme0n1.in 4.64453125$ disk.nvme0n1.out254.51025390625 disk.drbd2041.out0 disk.drbd1550.in0 disk.drbd1938.out0 disk.sda2.in0 disk.drbd1817.out0 disk.drbd1965.out0`AM2- _wandbruntime5259  _runtime5259D trainer/global_step11849 % _timestamp1.7858005467991276e+09 _runtime5259.622148094 _step944 ' lr-AdamW/pg10.00030537463766439196 ' lr-AdamW/pg20.00030537463766439196Z|{ % _timestamp1.7858005467991276e+09 _runtime5259.622148094 _step944 ' lr-AdamW/pg10.00030537463766439196 ' lr-AdamW/pg20.00030537463766439196 trainer/global_step11849f2- _wandbruntime5259  _runtime5259:zDO OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed203430667285504 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed214618918134784 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10426767877120 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10426680447488 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current241 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current239 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed80609522 trainer/global_step11849 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 203857250 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed78724127 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated78724129 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed10708759719936 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated10401248530940 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed213857347732992 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated80609763 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated208776852239772 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated198375603708832 _runtime5259.623299438 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed213857347732992 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 203866908 MFDeviceStatsMonitor.on_train_batcho_start/inactive_split.large_pool.peak18 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed198375586669472 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated214619071615488 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed203910158414848 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed214618918134784 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10426767877120 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10426680447488 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current241 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current239 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed80609522 trainer/global_step11849 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 203857250 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed78724127 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated78724129 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed10708759719936 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated10401248530940 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed213857347732992 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated80609763 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated208776852239772 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated198375603708832 _runtime5259.623299438 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed213857347732992 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 203866908 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed198375586669472 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated214619071615488 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed203910158414848 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 JDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 203884088 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 125153316 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current266 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed80616231 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 125153316 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 203874416 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed10427560067072 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated10427647503872 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 203874416 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 trainer/global_step11849 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated203447851087872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated78730772 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed203447834048512 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed46659782 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed198392326841792 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated46659784 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 203884088 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed208794367042830 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 125143646 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated203927390435328 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 _step946 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current264 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated33956713 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed02fM SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated10427647503872 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated213875498591744 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed203927365531648 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated213875498591744 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10427560067072 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated203447851087872 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated80616497 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated46659784 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 203884088 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed208794367042830 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 125143646 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated203927390435328 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 _step946 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current264 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated33956713 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated198392343881152 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed78730770 _runtime5259.993031129 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 203884088 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 125153316 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current266 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed80616231 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 125153316 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 203874416 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed10427560067072 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated10427647503872 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 203874416 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 trainer/global_step11849 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated203447851087872 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated78730772 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed203447834048512 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed46659782 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed198392326841792 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 TFDeviceStats Monitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 PDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10470676582912 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed10753919373312 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed214759682018816 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed209657646476796 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 % _timestamp1.7858005689139047e+09 _step949 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed79056277 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated46852697 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated80950142 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed10470676582912 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated10470764012544 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed10445051191324 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 _runtime5281.737849948 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated214759786487808 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated34097445 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed80949901 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10470764012544 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed204289005435904 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated204289022475264 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 204717402 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed79056277 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current241 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed215524433628160 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated10445136994448 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated209657749319280 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 204727060 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 trainer/global_step11899 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current239 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed199212595285472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated214759786487808 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed34097206 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated199212612324832 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 125670781 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed214759682018816 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated215524589206016 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated10754050047488 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 125670781 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated79056279 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated204289022475264 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed46852695 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 125661125 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated204770539158528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed204770514254848@'O ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10470676582912 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed10753919373312 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed214759682018816 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed209657646476796 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 % _timestamp1.7858005689139047e+09 _step949 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed79056277 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated46852697 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated80950142 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed10470676582912 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated10470764012544 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed10445051191324 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 _runtime5281.737849948 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated214759786487808 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated34097445 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed80949901 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10470764012544 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed204289005435904 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated204289022475264 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 204717402 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed79056277 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current241 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed215524433628160 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated10445136994448 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated209657749319280 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 204727060 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 trainer/global_step11899 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current239 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed199212595285472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated214759786487808 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed34097206 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated199212612324832 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 125670781 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed214759682018816 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated215524589206016 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated10754050047488 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 125670781 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated79056279 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated204289022475264 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed46852695 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 125661125 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated204770539158528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed204770514254848 2- _wandbruntime5282  _runtime52829ܕ M C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated46856634 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 204734568 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 125681318 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated80956929 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated199229352497152 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed79062920 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 204734568 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed46856632 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current260 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated209675366964934 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated79062922 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed214777728401408 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated34100295 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 204744240 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 204744240 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current262 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 _step950 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated204787746275328 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed204306172198912 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10471556202496 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated10446014467782 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed199229335457792 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed10445928664546 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated10471643639296 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 % _timestamp1.7858005692835834e+09 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated10471643639296 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 125671648 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed215542542825984 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated10754954218496 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed79062920 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed204306172198912 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed34100035 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed209675264122338 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed80956667M&M ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 125671648 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 % _timestamp1.7858005692835834e+09 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated10471643639296 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 125671648 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed215542542825984 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated10754954218496 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed79062920 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed204306172198912 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed34100035 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed209675264122338 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed80956667 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated46856634 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 204734568 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 125681318 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated80956929 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated199229352497152 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed79062920 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 204734568 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed46856632 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current260 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated209675366964934 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated79062922 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed214777728401408 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated34100295 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 204744240 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 204744240 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current262 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 _step950 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated204787746275328 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed204306172198912 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10471556202496 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated10446014467782 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed199229335457792 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed10445928664546 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated10471643639296 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed81290643 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed10799125712896 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 trainer/global_step11949 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 126198783 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _runtime5303.693561994 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed205630870094848 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed10488939654832 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10514760147968 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed47049545 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed210538543556304 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed200049603901472 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed79388427 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated205147360625664 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated10799262678528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated216430157677056 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed216429995807744 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated215662120773632 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated10514760147968 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed215662016304640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 126198783E:O ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed210538543556304 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed200049603901472 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed79388427 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated205147360625664 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated10799262678528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated216430157677056 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed216429995807744 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated215662120773632 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated10514760147968 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed215662016304640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 126198783 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed34241098 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 126189127 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated79388429 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated215662120773632 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed215662016304640 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated205630894998528 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated210538646398788 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed81290643 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed10799125712896 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 trainer/global_step11949 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 126198783 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _runtime5303.693561994 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed205630870094848 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed10488939654832 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10514760147968 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed47049545 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251md2- _wandbruntime5304  _runtime5304_T+&M NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated79395072 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed34243923 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated200066361113152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10800036174336 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 126209320 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed210556161201846 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated205648102115328 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 % _timestamp1.7858005912381604e+09 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 205594720 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated215680167163392 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed79395070 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 205604392 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 149279232 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 205594720 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated10489902931290 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed10489817128054 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed81297405 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed47053482 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated216448262665216 _runtime5304.061528319 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated34244163 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 205604392 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated10515639774720 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated79395072 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current242 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed210556161201846 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated205648102115328 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 % _timestamp1.7858005912381604e+09 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 205594720 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated215680167163392 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed79395070 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 205604392 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 149279232 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 205594720 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated10489902931290 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed10489817128054 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed81297405 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed47053482 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated216448262665216 _runtime5304.061528319 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated34244163 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 205604392 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated10515639774720 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated79395072 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current242 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated200066361113152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10800036174336 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 126209320 P(:  ݶV disk.drbd1285.in0 disk.drbd1584.out0 disk.drbd1673.in0 disk.drbd1550.in0 disk.drbd1289.in0 disk.sda.out 144.51171875 disk.sda1.in0 proc.cpu.threads29 disk.drbd1226.in0 disk.drbd1226.out0 disk.drbd2078.out0 disk.drbd1931.in0* proc.memory.percent17.182016372680664 disk.drbd2078.in0 disk.drbd1552.out0 cpu13.766797807560051 disk.drbd1285.out0! disk.dm-0.out255.65771484375 disk.drbd1817.in0 disk.drbd1673.out0! proc.memory.rssMB 11260.40625 disk.nvme0n1.in 4.64453125$ disk.nvme0n1.out255.65771484375 disk.drbd1817.out0 disk.rbd0.out0$ disk./.usageGB578.3483619689941 disk.dm-0.in 4.64453125 disk.drbd1356.in0 disk.drbd1356.out0 disk.drbd1938.in0 disk.drbd1938.out0 disk.sda1.out0$ memory_percent41.24886989593506) disk./.usagePercent32.34426802694515 disk.drbd1510.in0 disk.drbd2118.in0 disk.rbd1.out 21.3779296875 disk.drbd1965.out0 disk.drbd2041.in0 disk.drbd1552.in0 disk.drbd1828.in0 network.recv638731 disk.drbd1289.out0 disk.drbd1584.in0 disk.sda2.in0 disk.drbd1965.in0 disk.drbd2041.out0 disk.sda.in0 network.sent 1.1017095e+07 disk.drbd2111.in0 disk.drbd2111.out0 disk.rbd1.in0 disk.drbd1828.out0 disk.rbd0.in0 disk.drbd1510.out0 disk.drbd2118.out0 disk.sda2.out 144.51171875 disk.drbd1931.out0 disk.drbd1550.out0( proc.memory.availableMB 38503.140625\2- _wandbruntime5325  _runtime5325K _step956 & lr-AdamW/pg10.0003092404656371879 & lr-AdamW/pg20.0003092404656371879 trainer/global_step11999 % _timestamp1.7858006128874173e+09 _runtime5325.710421775b & lr-AdamW/pg10.0003092404656371879 & lr-AdamW/pg20.0003092404656371879 trainer/global_step11999 % _timestamp1.7858006128874173e+09 _runtime5325.710421775 _step956)u2- _wandbruntime5325  _runtime5325X''O UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed211419440635812 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated211419543478296 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated216564455059456 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 126717129 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated200886629556832 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed10532828118340 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated81631802 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed10844345309184 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed216564350590464 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 206437706 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed217335571244032 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated10532913921464 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed206005681736704 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated79720579 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 126726785 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed34385146 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current259 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10558668853760 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated216564455059456 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed10558668853760 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 126726785 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10558756283392 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 206447364 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed81631541 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated10844480177664 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed200886612517472 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 _runtime5325.714679017 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated206005698776064 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed206491225934848 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed79720577 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated34385405 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed206005681736704 _step957 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated10558756283392 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current261 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated206491250838528 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated79720579 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed79720577 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed216564350590464 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated217335731016192 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144'lDf'O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed10558668853760 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 126726785 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10558756283392 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 206447364 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed81631541 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated10844480177664 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed200886612517472 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 _runtime5325.714679017 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated206005698776064 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed206491225934848 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed79720577 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated34385405 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed206005681736704 _step957 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated10558756283392 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current261 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated206491250838528 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated79720579 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed79720577 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed216564350590464 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated217335731016192 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed211419440635812 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated211419543478296 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated216564455059456 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 126717129 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated200886629556832 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed10532828118340 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated81631802 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed10844345309184 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed216564350590464 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 206437706 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed217335571244032 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated10532913921464 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed206005681736704 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated79720579 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 126726785 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed34385146 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current259 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10558668853760 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated216564455059456 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392:2- _wandbruntime5326  _runtime5326&1.&M REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated79727222 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated216582501449216 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed34388052 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated216582501449216 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed217353682301440 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated206508457955328 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed206508433051648 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 206464544 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 206464544 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated206022865539072 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed216582396973056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 126737322 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 trainer/global_step11999 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed79727220 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current258 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated10533791394798 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated79727222 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 126727652 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current260 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed81638384 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated81638644 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10845249249792 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated216582501449216 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed217353682301440 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated206508457955328 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed206508433051648 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 206464544 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 206464544 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated206022865539072 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed216582396973056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 126737322 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 trainer/global_step11999 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed79727220 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current258 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated10533791394798 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated79727222 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 126727652 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current260 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed81638384 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated81638644 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10845249249792 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 EDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed217466684876288 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 127245131 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed10602664989184 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed201723621133472 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed47443245 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed80052727 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current268 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated10602752418816 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 trainer/global_step12049 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated206864036926464 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _step961 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 127254787 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed34528073 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed207351581774848 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed10889498397696 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10602752418816 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed206864019887104 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated34528339 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated207351606678528 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 _runtime5348.073732728 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 127245131 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated10889635363328 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated201723638172832 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed212300337715320 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated217466789345280 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated217466789345280 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated80052729 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed81971318 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 207307516 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed218241080172544 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 % _timestamp1.7858006352504134e+09 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated80052729 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated81971586 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated218241242041856 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed206864019887104 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated206864036926464 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated47443247>rn"O dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated10889635363328 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated201723638172832 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed212300337715320 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated217466789345280 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated217466789345280 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated80052729 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed81971318 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 207307516 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed218241080172544 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 % _timestamp1.7858006352504134e+09 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated80052729 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated81971586 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated218241242041856 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed206864019887104 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated206864036926464 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated47443247 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated10576802384972 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed217466684876288 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 127245131 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed10602664989184 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed201723621133472 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed47443245 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed80052727 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current268 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated10602752418816 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 trainer/global_step12049 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated206864036926464 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _step961 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 127254787 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed34528073 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed207351581774848 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed10889498397696 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 E?DeviceStatsMoniXAvtor.on_train_batch_start/oversize_segments.freed0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10602752418816 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed206864019887104 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated34528339 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated207351606678528 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 _runtime5348.073732728 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 127245131 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1y2- _wandbruntime5348  _runtime5348ë&M >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated80059372 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 127255654 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 207324696 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed10577594055070 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 % _timestamp1.7858006356262405e+09 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed206881186650112 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated212318058203458 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 207324696 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current237 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10890405049856 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed10603544608768 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated80059372 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed34530819 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _runtime 5348.44959782 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated217484835735040 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10603544608768 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed81978001 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed80059370 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 127265324 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed218259193941504 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 127255654 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated201740378345152 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 _step962 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 207315024 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated217484835735040 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current239 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated34531056 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed201740361305792 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed206881186650112 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 127265324 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated81978240 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed207368788891648 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed80059370 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0o&M ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed81978001 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed80059370 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 127265324 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed218259193941504 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 127255654 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated201740378345152 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 _step962 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 207315024 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated217484835735040 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current239 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated34531056 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed201740361305792 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed206881186650112 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 127265324 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated81978240 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed207368788891648 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed80059370 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated80059372 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 127255654 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 207324696 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed10577594055070 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 % _timestamp1.7858006356262405e+09 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed206881186650112 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated212318058203458 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 207324696 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current237 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10890405049856 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed10603544608768 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated80059372 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed34530819 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _runtime 5348.44959782 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated217484835735040 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10603544608768 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392.5h2- _wandbruntime5348  _runtime5348r?  _step963 2loss/SRqqt1/assignment_loss0.8651334801189867 /loss/FRt2/detection_loss2.2675698806460085 0loss/SRqqt2/detection_loss1.099056856351272 /loss/FBt2/detection_loss1.9159480390604586 &loss/total_loss16.573532005062113 epoch0 _runtime5348.450718095 2loss/SRqqt2/assignment_loss0.9290324259266783 .loss/FRt1/detection_loss2.490560554149674 trainer/global_step12049 4loss/FBt2/assignment_loss4.9529295768735436e-06 1loss/SRqqt1/detection_loss1.6658086672292225 0loss/FRt1/assignment_loss2.6420904354351324 /loss/FRt2/assignment_loss2.038112982725486 4loss/FBt1/assignment_loss3.7659147348020644e-07 /loss/FBt1/detection_loss0.6602133538981434 % _timestamp1.7858006356278882e+09) /loss/FRt2/assignment_loss2.038112982725486 4loss/FBt1/assignment_loss3.7659147348020644e-07 /loss/FBt1/detection_loss0.6602133538981434 % _timestamp1.7858006356278882e+09 _step963 2loss/SRqqt1/assignment_loss0.8651334801189867 /loss/FRt2/detection_loss2.2675698806460085 0loss/SRqqt2/detection_loss1.099056856351272 /loss/FBt2/detection_loss1.9159480390604586 &loss/total_loss16.573532005062113 epoch0 _runtime5348.450718095 2loss/SRqqt2/assignment_loss0.9290324259266783 .loss/FRt1/detection_loss2.490560554149674 trainer/global_step12049 4loss/FBt2/assignment_loss4.9529295768735436e-06 1loss/SRqqt1/detection_loss1.6658086672292225 0loss/FRt1/assignment_loss2.6420904354351324Uz](:  ǜV disk.rbd0.in0 disk.drbd2118.in0 disk.drbd2111.in0 disk.drbd1931.out0 disk.drbd1550.in0 disk.drbd1965.in0 disk.rbd0.out0 disk.drbd2111.out0% memory_percent41.248828172683716* proc.memory.percent17.182016372680664 disk.drbd1285.in0 disk.drbd1584.out0 disk.drbd2118.out0 disk.nvme0n1.in 4.64453125 disk.drbd1965.out0 disk.drbd1510.in0 disk.drbd2078.out0 disk.drbd1356.out0 disk.rbd1.out 21.5341796875 disk.drbd1828.in0 disk.drbd1938.in0 disk.sda.in0 disk.drbd1356.in0 disk.drbd1552.in0 disk.drbd1938.out0 disk.drbd1226.in0 disk.drbd1226.out0 disk.drbd2078.in0# disk.nvme0n1.out256.2373046875 disk.drbd1817.out0 disk.drbd1550.out0 disk.sda1.in0 cpu13.765528976016347$ disk./.usageGB578.3497505187988 disk.drbd2041.in0 disk.drbd1289.in0 disk.drbd1584.in0 disk.drbd1285.out0 disk.dm-0.out256.2373046875 disk.sda2.in0 disk.drbd1817.in0 disk.drbd1673.out0 disk.drbd1931.in0 proc.cpu.threads29 disk.sda2.out 145.75390625 disk.drbd1552.out0 disk.drbd1828.out0 disk.drbd1673.in0 disk.sda.out 145.75390625* proc.memory.availableMB38503.16796875! proc.memory.rssMB 11260.40625 disk.dm-0.in 4.64453125 disk.rbd1.in0 disk.drbd2041.out0 disk.sda1.out0 disk.drbd1289.out0 network.sent 1.1065464e+07 network.recv641488* disk./.usagePercent32.344345681919215 disk.drbd1510.out0~(: V gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts126.546' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent42.182 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0T%(: ㎳U gpu.0.gpu12.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts96.052' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent32.01733333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0gqZ(:  U disk.drbd1285.in0 disk.drbd2078.out0 disk.drbd1965.out0) disk./.usagePercent32.34434888198683 disk.rbd0.out0 disk.drbd2118.in0 network.recv642859 disk.drbd1938.out0 proc.cpu.threads29 disk.drbd1550.in0 disk.rbd0.in0 disk.drbd1510.in0 disk.drbd1938.in0 disk.sda1.out0 disk.drbd2111.out0 disk.sda.in0 disk.drbd1289.in0 disk.drbd1356.out0 disk.rbd1.in0 disk.drbd1673.in0 disk.drbd1965.in0 disk.drbd1931.out0 disk.drbd1552.in0* proc.memory.availableMB38502.41796875 network.sent 1.1110419e+07 disk.drbd1289.out0 disk.dm-0.in 4.64453125 disk.drbd1817.out0 disk.rbd1.out 21.5341796875 disk.sda.out 145.90234375 disk.drbd2118.out0 disk.sda2.in0 disk.sda2.out 145.90234375 disk.drbd1931.in0 disk.drbd2041.in0 disk.drbd1285.out0 disk.drbd1226.out0 disk.drbd2078.in0 disk.drbd1817.in0 disk.drbd1673.out0 disk.drbd2041.out0$ disk./.usageGB578.3498077392578 disk.drbd1510.out0 disk.dm-0.out256.4365234375 disk.drbd1226.in0# disk.nvme0n1.out256.4365234375* proc.memory.percent17.182016372680664 cpu13.764843551211225 disk.drbd1584.out0 disk.drbd1550.out0! proc.memory.rssMB 11260.40625 disk.sda1.in0 disk.drbd1584.in0 disk.nvme0n1.in 4.64453125 disk.drbd1356.in0 disk.drbd2111.in0 disk.drbd1828.in0# memory_percent41.2499725818634 disk.drbd1552.out0 disk.drbd1828.out0]2- _wandbruntime5370  _runtime5370n $ _timestamp1.785800657344495e+09 _runtime5370.167456455 _step964 & lr-AdamW/pg10.0003118176842857185 & lr-AdamW/pg20.0003118176842857185 trainer/global_step12099}&uA _runtime5370.167456455 _step964 & lr-AdamW/pg10.0003118176842857185 & lr-AdamW/pg20.0003118176842857185 trainer/global_step12099 $ _timestamp1.785800657344495e+09l2- _wandbruntime5370  _runtime5370XgO QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed80384877 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed10934718333440 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated202560646788832 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 127773133 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 208158010 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10646661124608 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 208167668 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed10646661124608 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 127782789 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed207722358037504 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed219146655948288 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed82310256 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed47640095 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated218369123631104 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 _runtime5370.168724625 _step965 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10646748554240 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed218369019162112 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated10620690848480 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated47640097 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated10646748554240 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated208211962518528 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 trainer/global_step12099 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed218369019162112 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated207722375076864 J'O ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 127773133 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated207722375076864 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed202560629749472 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated219146819914752 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated213181337637312 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed80384877 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 127782789 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed10620605045356 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated218369123631104 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated82310513 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed213181234794828 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed207722358037504 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated80384879 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed218369019162112 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated10620690848480 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated47640097 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated10646748554240 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated208211962518528 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 trainer/global_step12099 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed218369019162112 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated207722375076864 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 127782789 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed207722358037504 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed219146655948288 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed82310256 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed47640095 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated218369123631104 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 _runtime5370.168724625 _step965 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10646748554240 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated10647628180992 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 127783656 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated10647628180992 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed202577369921792 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime5370.541825875 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed207739524800512 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed80391520 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed207739524800512 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated213198955282966 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed47644032 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 151376384 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 trainer/global_step12099 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 208175176 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10647540744192 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 127793326 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed82316988 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated207739541839872 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed10647540744192 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated80391522 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 208175176 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current257 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated208229169635328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 126472704 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated207739541839872 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current259 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed218387065544704 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated219164924124672 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated10935754489344 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated10621568321814 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed10621482518578 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 208184848 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated202577386961152 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _step966 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 208184848 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated218387170020864 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated34673213 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670i8%M ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 208184848 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated202577386961152 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _step966 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 208184848 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated218387170020864 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated34673213 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated10647628180992 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 127783656 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated10647628180992 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed202577369921792 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime5370.541825875 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed207739524800512 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed80391520 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed207739524800512 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated213198955282966 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed47644032 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 151376384 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 trainer/global_step12099 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 208175176 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10647540744192 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 127793326 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed82316988 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated207739541839872 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed10647540744192 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated80391522 H:DeviceStatsMonitor.oEZ  n_train_batch_end/allocation.all.freed 208175176 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current257 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated208229169635328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 126472704 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated207739541839872 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current259 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed218387065544704 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated219164924124672 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated10935754489344 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated10621568321814 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed10621482518578 FDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed219271353447936 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed208580696187904 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed219271353447936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 209018162 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated208580713227264 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _runtime5392.397158003 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 trainer/global_step12149 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated219271457916928 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _step969 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated80717029 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 128301135 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated82649656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed34812473 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed203397638365472 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed82649418 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 128310791 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10690744689664 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated34812709 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated220052317979648 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed209072293454848 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated203397655404832 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 128301135 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10690657260032 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated214062234716820 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated10979999621120 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current238 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated208580713227264 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed80717027 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 209027820 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392u'O C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed219271353447936 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed208580696187904 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed219271353447936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 209018162 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated208580713227264 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _runtime5392.397158003 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 trainer/global_step12149 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated219271457916928 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _step969 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated80717029 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 128301135 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated82649656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed34812473 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed203397638365472 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed82649418 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 128310791 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10690744689664 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated34812709 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated220052317979648 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed209072293454848 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated203397655404832 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 128301135 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10690657260032 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated214062234716820 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated10979999621120 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current238 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated208580713227264 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed80717027 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 209027820 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed208580696187904 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated47836947 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated209072318358528 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed214062131874336 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0ye2- _wandbruntime5392  _runtime5392c#hgM J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _runtime5392.761871235 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed82656138 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated208597879990272 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 209035328 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current273 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed47840882 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 209045000 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated209089525475328 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed34815256 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated203414395577152 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 209035328 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 128311658 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated10665456785322 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed10665370982086 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated220070423926272 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10980757298176 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 trainer/global_step12149 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated219289504306688 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed80723670 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed209089500571648 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated80723672 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E=DeviceS}3tatsMonitor.on_train_batch_end/inactive_split.all.peak548 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated80723672 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated82656411 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated214079852362474 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated10691624316416 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed220070257869824 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed214079749519878 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated47840884 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 128321328 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated208597879990272 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current271 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed82656138 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated208597879990272 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 209035328 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current273 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed47840882 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 209045000 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated209089525475328 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed34815256 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated203414395577152 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 209035328 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 128311658 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated10665456785322 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed10665370982086 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated220070423926272 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed10980757298176 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 trainer/global_step12149 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated219289504306688 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed80723670 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed209089500571648 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated80723672 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated80723672 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated82656411 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated214079852362474 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated10691624316416 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed220070257869824 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed214079749519878 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated47840884 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 128321328 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak00)2- _wandbruntime5392  _runtime5392  0loss/SRqqt2/detection_loss1.228163793530257 /loss/FBt1/detection_loss0.6093221330083907 /loss/FBt2/detection_loss2.0871889049187304 _step971 3loss/FBt2/assignment_loss2.384185648907078e-08 /loss/FRt1/detection_loss2.6555079070778635 %loss/total_loss17.92959337111067 epoch0 0loss/FRt1/assignment_loss2.6739662498622745 2loss/SRqqt2/assignment_loss0.8337718307639079 3loss/FBt1/assignment_loss6.443744612969002e-09 0loss/SRqqt1/detection_loss1.698992382735014 0loss/FRt2/assignment_loss2.4111403392021913 2loss/SRqqt1/assignment_loss1.2541185353042148 /loss/FRt2/detection_loss2.4774212644222273 trainer/global_step12149 % _timestamp1.7858006799401884e+09 _runtime5392.763013841S 3loss/FBt2/assignment_loss2.384185648907078e-08 /loss/FRt1/detection_loss2.6555079070778635 %loss/total_loss17.92959337111067 epoch0 0loss/FRt1/assignment_loss2.6739662498622745 2loss/SRqqt2/assignment_loss0.8337718307639079 3loss/FBt1/assignment_loss6.443744612969002e-09 0loss/SRqqt1/detection_loss1.698992382735014 0loss/FRt2/assignment_loss2.4111403392021913 2loss/SRqqt1/assignment_loss1.2541185353042148 /loss/FRt2/detection_loss2.4774212644222273 trainer/global_step12149 % _timestamp1.7858006799401884e+09 _runtime5392.763013841 0loss/SRqqt2/detection_loss1.228163793530257 /loss/FBt1/detection_loss0.6093221330083907 /loss/FBt2/detection_loss2.0871889049187304 _step971Z(: U gpu.0.gpu38.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts96.246' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent32.082 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0ۍ9Y(:  ۫U network.recv645304 disk.drbd1828.out0 disk.drbd1289.in0 disk.drbd1938.in0 disk.drbd1356.out0 cpu13.763620058710057 disk.drbd1931.out0 disk.drbd1938.out0 disk.sda2.in0 disk.drbd1584.out0 network.sent 1.1158442e+07* disk./.usagePercent32.344349308662515 disk.drbd1965.out0 disk.drbd1550.out0 disk.drbd2041.in0 disk.drbd1828.in0 disk.drbd2111.out0$ memory_percent41.25012159347534 disk.drbd1817.in0 disk.drbd1817.out0 disk.sda1.out0 disk.dm-0.in 4.64453125 disk.drbd2111.in0* proc.memory.percent17.182016372680664 disk.drbd1510.in0 disk.drbd1226.out0 disk.drbd1584.in0 disk.rbd1.out 21.74609375 disk.drbd1965.in0 disk.sda.in0 disk.drbd2078.out0$ disk./.usageGB578.3498153686523 disk.drbd2041.out0 disk.drbd1510.out0 disk.drbd1226.in0 disk.drbd2118.in0 disk.drbd1285.out0 disk.drbd2118.out0! disk.dm-0.out257.97021484375) proc.memory.availableMB 38502.3203125 disk.drbd1931.in0 disk.drbd1289.out0 disk.rbd0.in0 disk.sda.out 146.6015625! proc.memory.rssMB 11260.40625 proc.cpu.threads29$ disk.nvme0n1.out257.97021484375 disk.drbd1356.in0 disk.sda2.out 146.6015625 disk.drbd1673.in0 disk.rbd1.in0 disk.drbd1552.in0 disk.drbd1673.out0 disk.drbd1552.out0 disk.drbd1285.in0 disk.drbd2078.in0 disk.rbd0.out0 disk.drbd1550.in0 disk.sda1.in0 disk.nvme0n1.in 4.64453125<:2- _wandbruntime5414  _runtime54149G % _timestamp1.7858007015286748e+09 _runtime5414.351652162 _step972 & lr-AdamW/pg10.0003143949029342491 & lr-AdamW/pg20.0003143949029342491 trainer/global_step12199 9 & lr-AdamW/pg10.0003143949029342491 & lr-AdamW/pg20.0003143949029342491 trainer/global_step12199 % _timestamp1.7858007015286748e+09 _runtime5414.351652162 _step972;2- _wandbruntime5414  _runtime5414 'O aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed209932649294848 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated10708467775496 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated209439051377664 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current256 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10734740825088 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed214943028953844 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated81049179 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed209439034338304 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed34954407 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 trainer/global_step12199 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated220173792202752 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed81049177 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11025131053056 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 128829137 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed220957645479424 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed204234646981472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed209439034338304 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 209878314 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed11024996184576 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 128829137 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed220173687733760 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated34954661 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 128829137 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ZFDeUk9viceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed220957645479424 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed204234646981472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed209439034338304 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 209878314 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed11024996184576 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 128829137 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed220173687733760 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated34954661 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 209895480 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated82995262 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed48037732 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated34957528 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated209949881315328 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed220975754120192 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 128849330 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 128849330 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed81055820 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 _runtime 5414.72826582 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated220191838592512 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed10735533015040 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current249 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated204251404193152 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 % _timestamp1.7858007019049056e+09 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 209905152 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed209456201101312 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 209895480 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10735533015040 trainer/global_step12199 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 128839660 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed209456201101312 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11025897708544 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated214960749441982 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed204251387153792 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed34957279 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed209949856411648 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed10709259445594 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed82995011 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated48037734 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed214960646599386 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251S&M QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 _runtime 5414.72826582 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated220191838592512 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed10735533015040 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current249 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated204251404193152 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 % _timestamp1.7858007019049056e+09 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 209905152 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed209456201101312 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 209895480 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10735533015040 trainer/global_step12199 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 128839660 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed209456201101312 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11025897708544 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated214960749441982 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed204251387153792 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed34957279 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed209949856411648 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed10709259445594 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed82995011 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated48037734 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed214960646599386 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated81055822 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated209456218140672 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 209895480 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated82995262 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed48037732 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated34957528 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated209949881315328 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed220975754120192 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 128849330 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 128849330 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed81055820 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251+2- _wandbruntime5414  _runtime5414G  4loss/FBt1/assignment_loss1.3754916658661806e-08 1loss/SRqqt2/detection_loss1.0626440557165884 _step975 0loss/FRt2/assignment_loss2.4988809455382195 2loss/SRqqt1/assignment_loss0.7318258677795529 2loss/SRqqt2/assignment_loss0.9539139322374134 ,loss/FRt1/detection_loss2.4918872964235 1loss/SRqqt1/detection_loss1.6090413384139537 -loss/FBt2/detection_loss2.83663537632674 &loss/total_loss18.254833853260287 $ _timestamp1.785800701906779e+09 0loss/FRt1/assignment_loss2.8075116365044206 4loss/FBt2/assignment_loss7.5814234151039274e-06 /loss/FBt1/detection_loss0.8976301082338278 _runtime5414.729605843 /loss/FRt2/detection_loss2.3648557009077384 epoch0 trainer/global_step12199 4loss/FBt1/assignment_loss1.3754916658661806e-08 1loss/SRqqt2/detection_loss1.0626440557165884 _step975 0loss/FRt2/assignment_loss2.4988809455382195 2loss/SRqqt1/assignment_loss0.7318258677795529 2loss/SRqqt2/assignment_loss0.9539139322374134 ,loss/FRt1/detection_loss2.4918872964235 1loss/SRqqt1/detection_loss1.6090413384139537 -loss/FBt2/detection_loss2.83663537632674 &loss/total_loss18.254833853260287 $ _timestamp1.785800701906779e+09 0loss/FRt1/assignment_loss2.8075116365044206 4loss/FBt2/assignment_loss7.5814234151039274e-06 /loss/FBt1/detection_loss0.8976301082338278 _runtime5414.729605843 /loss/FRt2/detection_loss2.3648557009077384 epoch0 trainer/global_step12199(: U gpu.0.gpu28.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts102.104' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent34.034666666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors00[(:  U network.sent 1.1203432e+07 disk.drbd2041.in0 network.recv646864 disk.drbd1289.in0 disk.drbd1584.out0 disk.rbd1.in0% memory_percent41.250479221343994 disk.sda.out 147.18359375 disk.drbd1285.out0 disk.drbd1965.out0 proc.cpu.threads29 disk.drbd1931.in0 disk.sda1.out0 disk.dm-0.in 4.64453125$ disk.nvme0n1.out258.98193359375 disk.drbd1356.out0 disk.drbd1938.out0 disk.drbd2118.in0 disk.drbd1817.in0 disk.nvme0n1.in 4.64453125 disk.drbd1584.in0 disk.drbd2111.in0 disk.rbd1.out 21.80859375) proc.memory.availableMB 38502.0859375* proc.memory.percent17.182016372680664 disk.drbd1931.out0 disk.drbd2111.out0 disk.drbd1673.out0 disk.drbd1817.out0 disk.drbd1828.out0 disk.drbd1550.in0 disk.drbd1550.out0 disk.drbd1938.in0 disk.drbd1965.in0* disk./.usagePercent32.344352508730125 disk.rbd0.in0 disk.rbd0.out0 disk.drbd2118.out0 disk.sda2.in0 disk.drbd2041.out0 disk.drbd1285.in0 disk.drbd1510.in0! proc.memory.rssMB 11260.40625 disk.drbd1552.out0 disk.sda2.out 147.18359375$ disk./.usageGB578.3498725891113 disk.drbd1289.out0 disk.sda1.in0 disk.drbd1510.out0 disk.drbd2078.in0 disk.drbd1356.in0 cpu13.76301203260547 disk.drbd1673.in0 disk.sda.in0! disk.dm-0.out258.98193359375 disk.drbd1226.in0 disk.drbd1226.out0 disk.drbd1552.in0 disk.drbd1828.in0 disk.drbd2078.out0t(: ՌU gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts101.626' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent33.87533333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0%Y(:  U disk.sda.out 147.68359375 disk.drbd1356.in0! proc.memory.rssMB 11260.40625$ disk./.usageGB578.3480529785156 disk.sda2.in0 disk.drbd2111.out0 disk.drbd1965.in0 disk.drbd1584.out0 disk.sda2.out 147.68359375 disk.drbd1673.out0 disk.drbd2041.out0 disk.drbd1828.in0! disk.dm-0.out259.25927734375 network.sent 1.1248454e+07 disk.drbd1828.out0 disk.drbd1584.in0 disk.drbd1356.out0$ memory_percent41.25016927719116 disk.drbd1938.in0 disk.drbd1285.out0) disk./.usagePercent32.34425074658004 disk.drbd2041.in0 disk.drbd1289.in0* proc.memory.percent17.182016372680664 disk.drbd1965.out0 disk.drbd1938.out0 disk.rbd0.in0 disk.sda1.out0 disk.drbd1510.out0 disk.drbd1226.out0$ disk.nvme0n1.out259.25927734375 disk.drbd1817.out0 disk.drbd2078.out0 disk.drbd2118.in0 disk.drbd2118.out0 disk.drbd1931.out0 disk.sda.in0 disk.drbd1552.out0 disk.drbd1550.in0 disk.drbd1226.in0 network.recv648494 disk.drbd1931.in0 disk.drbd1550.out0 disk.drbd1289.out0 disk.dm-0.in 4.64453125 disk.nvme0n1.in 4.64453125 proc.cpu.threads29 disk.drbd1673.in0 disk.rbd1.in0 disk.rbd1.out 21.80859375 disk.drbd1552.in0 disk.rbd0.out0 disk.drbd1285.in0) proc.memory.availableMB 38502.2890625 cpu13.76245472574441 disk.drbd1817.in0 disk.sda1.in0 disk.drbd1510.in0 disk.drbd2078.in0 disk.drbd2111.in02- _wandbruntime5436  _runtime54365 & lr-AdamW/pg10.0003156835122585144 & lr-AdamW/pg20.0003156835122585144 trainer/global_step12249 % _timestamp1.7858007239067857e+09 _runtime5436.731775112 _step976W _runtime5436.731775112 _step976 & lr-AdamW/pg10.0003156835122585144 & lr-AdamW/pg20.0003156835122585144 trainer/global_step12249 % _timestamp1.7858007239067857e+09Z2- _wandbruntime5436  _runtime5436~ѢO O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 YEPK{<'DeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed210297372488704 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 $ _timestamp1.785800723907456e+09 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed11070151825920 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated81381329 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed35098625 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated210297389528064 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current247 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated210297389528064 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed83329270 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed210793005134848 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated48230647 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated205071672636832 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated221076126488576 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed215823926033352 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed48230645 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11070288791552 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated10752356239004 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 _runtime5436.731859076 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated221076126488576 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed221076022019584 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated81381329 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated210793030038528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed10752270435880 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current249 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed205071655597472 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed81381327 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 129357139 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated35098872 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 trainer/global_step12249 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 129366795 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed10778649530880 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed81381327 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated10778736960512 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated221863318830080 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed210297372488704 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated210297389528064 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed83329270 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed210793005134848 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated48230647 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated205071672636832 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated221076126488576 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed215823926033352 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed48230645 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11070288791552 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated10752356239004 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 _runtime5436.731859076 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated221076126488576 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed221076022019584 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated81381329 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated210793030038528 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed10752270435880 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current249 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed205071655597472 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed81381327 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 129357139 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated35098872 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 trainer/global_step12249 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 129366795 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0x2- _wandbruntime5437  _runtime5437DX&M NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed81387970 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 210755632 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed10753147909102 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 _runtime5437.103271923 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11071188071936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated10779616587264 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed221881269656576 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed221094068402176 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated215841646521490 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed10779529150464 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10779529150464 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated35101704 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated210314556291072 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current247 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed48234582 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated221094172878336 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated210314556291072 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated81387972 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 210765304 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current245 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated81387972 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated210810237155328 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed215841543678894 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 % _timestamp1.7858007242797866e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated10753233712338 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed35101459 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed81387970 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed205088395769792 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 129377332 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed83336041 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated10779616587264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated48234584 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated221881425227264 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 210755632 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 210765304 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated205088412809152 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated221094172878336 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated83336288 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 trainer/global_step12249 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 129377332 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 % _timestamp1.7858007242797866e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated10753233712338 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed35101459 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed81387970 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed205088395769792 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 129377332 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed83336041 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated10779616587264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated48234584 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated221881425227264 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 210755632 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 210765304 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated205088412809152 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 LDDevȈKiceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated221094172878336 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated83336288 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 trainer/global_step12249 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 129377332 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated210314556291072 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current247 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed48234582 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated221094172878336 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated210314556291072 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated81387972 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 210765304 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current245 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated81387972 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated210810237155328 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed215841543678894b]<2- _wandbruntime5437  _runtime5437M  0loss/FRt1/assignment_loss2.5856998119805312 0loss/FRt2/assignment_loss2.4882706591957495 0loss/SRqqt2/detection_loss0.899754959155495 $ _timestamp1.785800724281136e+09 /loss/FRt1/detection_loss2.6134688000421264 /loss/FRt2/detection_loss2.3712842814232173 epoch0 3loss/FBt2/assignment_loss9.934106870446158e-09 1loss/SRqqt1/detection_loss1.6166880642590316 /loss/FBt2/detection_loss1.8025054937849443 _runtime5437.103988984 _step979 2loss/SRqqt1/assignment_loss0.9474584140268195 2loss/SRqqt2/assignment_loss1.0017067035093594 4loss/FBt1/assignment_loss2.2076979649909844e-06 /loss/FBt1/detection_loss0.7198718672245741 %loss/total_loss17.04671127223392 trainer/global_step122492C 2loss/SRqqt1/assignment_loss0.9474584140268195 2loss/SRqqt2/assignment_loss1.0017067035093594 4loss/FBt1/assignment_loss2.2076979649909844e-06 /loss/FBt1/detection_loss0.7198718672245741 %loss/total_loss17.04671127223392 trainer/global_step12249 0loss/FRt1/assignment_loss2.5856998119805312 0loss/FRt2/assignment_loss2.4882706591957495 0loss/SRqqt2/detection_loss0.899754959155495 $ _timestamp1.785800724281136e+09 /loss/FRt1/detection_loss2.6134688000421264 /loss/FRt2/detection_loss2.3712842814232173 epoch0 3loss/FBt2/assignment_loss9.934106870446158e-09 1loss/SRqqt1/detection_loss1.6166880642590316 /loss/FBt2/detection_loss1.8025054937849443 _runtime5437.103988984 _step979/סޚ(: U gpu.0.gpu14.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts96.052' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent32.01733333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0wI\(:  U disk.drbd2078.in0 disk.drbd2118.in0 disk.drbd2118.out0 disk.drbd1938.in0 disk.drbd1938.out0 disk.sda.in0 disk.dm-0.in 4.64453125 disk.drbd1226.in0 disk.sda2.in0 disk.drbd1289.in0 disk.sda.out 147.94921875 disk.drbd1356.in0 disk.drbd1550.out0 disk.drbd1289.out0 disk.rbd0.in0 disk.rbd0.out0 disk.drbd1510.out0 proc.cpu.threads29$ disk./.usageGB578.3499717712402 disk.drbd2111.in0 disk.drbd1673.out0 disk.drbd2041.in0 disk.drbd1285.out0 disk.drbd1673.in0 disk.drbd1931.out0* disk./.usagePercent32.344358055513986 disk.drbd2078.out0 disk.drbd1584.out0 disk.dm-0.out259.6787109375* proc.memory.availableMB38502.47265625 disk.drbd2041.out0 disk.rbd1.out 21.8408203125 disk.drbd1552.in0 disk.drbd1828.out0 disk.drbd1584.in0 disk.drbd1817.in0 disk.rbd1.in0 disk.sda1.in0$ memory_percent41.24988913536072# disk.nvme0n1.out259.6787109375 disk.drbd1356.out0 disk.sda2.out 147.94921875 disk.drbd1931.in0 disk.drbd1550.in0 cpu13.761715857164091 network.recv649376 disk.drbd1552.out0* proc.memory.percent17.182016372680664! proc.memory.rssMB 11260.40625 disk.drbd2111.out0 disk.drbd1817.out0 disk.drbd1965.in0 disk.sda1.out0 disk.drbd1226.out0 disk.drbd1965.out0 disk.nvme0n1.in 4.64453125 disk.drbd1285.in0 network.sent 1.1251517e+07 disk.drbd1828.in0 disk.drbd1510.in0V2- _wandbruntime5458  _runtime5458 _runtime5458.467445115 _step980 & lr-AdamW/pg10.0003169721215827797 & lr-AdamW/pg20.0003169721215827797 trainer/global_step12299 % _timestamp1.7858007456443505e+09Z & lr-AdamW/pg20.0003169721215827797 trainer/global_step12299 % _timestamp1.7858007456443505e+09 _runtime5458.467445115 _step980 & lr-AdamW/pg10.0003169721215827797C2- _wandbruntime5458  _runtime5458 2'O SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current274 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated222768816481280 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 129894797 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed205908664213472 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated35243962 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed35243690 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated211653385878528 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11115430602752 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 211598618 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated205908681252832 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed11115293637120 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed10796158899388 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed221978356305408 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed222768654611968 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed81713477 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 129885141 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed211155710639104 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10822733095936 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10822645666304 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 211608276 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated81713479 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 _step981 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed48427495 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated211155727678464 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated216704925955344 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated10796244702512 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 % _timestamp1.7858007456455247e+09 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed211653360974848<1'O TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated81713479 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 _step981 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed48427495 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated211155727678464 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated216704925955344 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated10796244702512 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 % _timestamp1.7858007456455247e+09 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed211653360974848 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current274 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated222768816481280 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 129894797 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed205908664213472 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated35243962 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed35243690 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated211653385878528 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11115430602752 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 211598618 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated205908681252832 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed11115293637120 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed10796158899388 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed221978356305408 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed222768654611968 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed81713477 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 129885141 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed211155710639104 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10822733095936 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10822645666304 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 211608276 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current1703936032- _wandbruntime5458  _runtime5458N"M QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small4$_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated216722543600998 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed83677932 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed221996402688000 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed81720120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed211172877402112 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 211615784 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed222786767037440 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed216722440758402 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed81720120 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11116329612800 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 211625456 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 129895664 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated221996507164160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed211670568091648 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11116198945792 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 _step982 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated10823612722688 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 $ _timestamp1.785800746011897e+09 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10823525285888 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 211625456 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed48431432 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 trainer/global_step12299 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 129905334 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 129895664 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed205925404385792 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 129905334 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated48431434 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed211172877402112 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 _runtime 5458.83621301 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated221996507164160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated35246749 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 211615784 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated81720122 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated81720122 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed10823525285888 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed10797036372610 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888!L&M M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed81720120 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11116329612800 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 211625456 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 129895664 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated221996507164160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed211670568091648 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11116198945792 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 _step982 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated10823612722688 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 $ _timestamp1.785800746011897e+09 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10823525285888 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 211625456 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed48431432 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 trainer/global_step12299 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 129905334 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 129895664 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed205925404385792 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 129905334 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated48431434 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed211172877402112 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 _runtime 5458.83621301 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated221996507164160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated35246749 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 211615784 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated81720122 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated81720122 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed10823525285888 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed10797036372610 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated216722543600998 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed83677932 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed221996402688000 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed81720120 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed211172877402112 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 211615784 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed222786767037440 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed216722440758402 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160d-2- _wandbruntime5458  _runtime5458XGo  &loss/total_loss16.064910052635923 _step983 /loss/FRt2/assignment_loss2.259689109772444 /loss/FBt1/detection_loss0.6395682647926151 epoch0 % _timestamp1.7858007460134358e+09 _runtime5458.836476188 2loss/SRqqt2/assignment_loss0.8527371166405123 trainer/global_step12299 0loss/FRt1/assignment_loss2.5945365884330833 2loss/SRqqt1/assignment_loss0.7106055252406062 /loss/FRt1/detection_loss2.4750355224261122 1loss/SRqqt2/detection_loss0.9369856356380796 2loss/FBt1/assignment_loss0.011157918721425464 3loss/FBt2/assignment_loss6.675709300907329e-07 /loss/FRt2/detection_loss2.3996032997965813 0loss/SRqqt1/detection_loss1.568050834711413 /loss/FBt2/detection_loss1.6169395688921213fK 2loss/FBt1/assignment_loss0.011157918721425464 3loss/FBt2/assignment_loss6.675709300907329e-07 /loss/FRt2/detection_loss2.3996032997965813 0loss/SRqqt1/detection_loss1.568050834711413 /loss/FBt2/detection_loss1.6169395688921213 &loss/total_loss16.064910052635923 _step983 /loss/FRt2/assignment_loss2.259689109772444 /loss/FBt1/detection_loss0.6395682647926151 epoch0 % _timestamp1.7858007460134358e+09 _runtime5458.836476188 2loss/SRqqt2/assignment_loss0.8527371166405123 trainer/global_step12299 0loss/FRt1/assignment_loss2.5945365884330833 2loss/SRqqt1/assignment_loss0.7106055252406062 /loss/FRt1/detection_loss2.4750355224261122 1loss/SRqqt2/detection_loss0.9369856356380796bh(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts127.468' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent42.489333333333335 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0^Y(:  U disk.sda1.out0 disk.drbd1285.in0 disk.drbd2111.in0 disk.drbd1965.out0 disk.rbd0.in0 disk.drbd1510.out0 disk.drbd1552.in0! proc.memory.rssMB 11260.40625 disk.drbd1356.out0 disk.rbd0.out0 disk.sda.out 148.24609375 disk.drbd1226.in0# disk.nvme0n1.out259.8896484375 disk.dm-0.in 4.64453125 disk.drbd2118.in0 disk.drbd1552.out0 disk.sda1.in0 disk.drbd1938.out0$ disk./.usageGB578.3499526977539 disk.drbd1673.in0 disk.drbd1828.out0 disk.drbd1285.out0 disk.drbd1584.out0 disk.nvme0n1.in 4.64453125 disk.sda2.in0 disk.drbd1931.in0 disk.drbd1550.out0 network.recv650771* disk./.usagePercent32.344356988824785 disk.drbd1817.out0 disk.drbd1828.in0% memory_percent41.249942779541016 proc.cpu.threads29 disk.drbd2041.in0 disk.drbd1550.in0 disk.sda.in0 disk.drbd1510.in0 disk.drbd2078.out0 disk.drbd1584.in0 disk.sda2.out 148.24609375 disk.drbd1965.in0 disk.drbd1817.in0 cpu13.761255947727202 network.sent 1.1296483e+07 disk.rbd1.in0* proc.memory.percent17.182016372680664 disk.drbd2078.in0 disk.drbd1931.out0 disk.drbd1289.in0& proc.memory.availableMB 38502.4375 disk.drbd2118.out0 disk.drbd1356.in0 disk.drbd2111.out0 disk.drbd1673.out0 disk.drbd1289.out0 disk.dm-0.out259.8896484375 disk.drbd1226.out0 disk.rbd1.out 22.0283203125 disk.drbd2041.out0 disk.drbd1938.in03(: U gpu.0.gpu24.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts100.74' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent33.58 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0RHY(:  U disk.drbd1965.in0 disk.drbd1552.in0 disk.drbd1510.in0 disk.drbd2118.in0 disk.drbd1550.in0 disk.drbd1938.out0 disk.rbd0.in0 disk.drbd1285.in0 disk.dm-0.out 260.271484375 disk.sda2.in0 disk.drbd2111.in0" disk.nvme0n1.out 260.271484375 disk.drbd1965.out0 disk.drbd2041.out0 disk.sda1.out0 disk.drbd1356.out0) disk./.usagePercent32.34435378875717 disk.rbd1.in0 disk.sda2.out 148.81640625 disk.drbd1289.out0 disk.rbd0.out0 disk.sda.out 148.81640625 disk.drbd1584.in0 disk.drbd1931.in0 network.sent 1.1341547e+07 disk.drbd1817.in0 disk.drbd1552.out0 cpu13.760683148347923 disk.rbd1.out 22.0908203125 disk.drbd1673.out0 disk.drbd1510.out0 disk.drbd1356.in0! proc.memory.rssMB 11260.40625* proc.memory.percent17.182016372680664 proc.cpu.threads29 disk.drbd1828.in0 disk.drbd1289.in0 disk.dm-0.in 4.64453125 disk.drbd2111.out0 network.recv652361 disk.drbd2041.in0 disk.sda.in0 disk.drbd1226.out0 disk.drbd2078.out0* proc.memory.availableMB38501.20703125 disk.drbd2118.out0 disk.drbd1938.in0$ disk./.usageGB578.3498954772949 disk.drbd1673.in0 disk.drbd1550.out0 disk.drbd1226.in0 disk.drbd2078.in0 disk.drbd1931.out0 disk.sda1.in0 disk.drbd1285.out0 disk.nvme0n1.in 4.64453125 disk.drbd1817.out0 disk.drbd1828.out0 disk.drbd1584.out0$ memory_percent41.251820325851442- _wandbruntime5480  _runtime5480B _step984 % lr-AdamW/pg10.000318260730907045 % lr-AdamW/pg20.000318260730907045 trainer/global_step12349 % _timestamp1.7858007679074335e+09 _runtime5480.730433232 bN trainer/global_step12349 % _timestamp1.7858007679074335e+09 _runtime5480.730433232 _step984 % lr-AdamW/pg10.000318260730907045 % lr-AdamW/pg20.000318260730907045o1=2- _wandbruntime5480  _runtime54809O \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed82045627 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated223674392769024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed206745672829472 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed10866641801728 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated84010736 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed35386137 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed223674230899712 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed82045627 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated222880795060224 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11160651050496 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed212014048789504 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10866641801728 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated217585823034852 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated82045629 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated212014065828864 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed48624345 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated48624347 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed212513716814848 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 130422799 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated82045629 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 % _timestamp1.7858007679084408e+09 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed222880690591232 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed212014048789504 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 _runtime5480.731661501 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocate1 d 212468428 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed222880690591232 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed217585720192368 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated10840133166020 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current254 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated212513741718528 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated35386389 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed10840047362896 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed84010482 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 130422799 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 130413143 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 130413143 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current252 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated212014065828864 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10866729231360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0"{'O QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 130422799 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated82045629 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 % _timestamp1.7858007679084408e+09 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed222880690591232 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed212014048789504 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 _runtime5480.731661501 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 212468428 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed222880690591232 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed217585720192368 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated10840133166020 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current254 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated212513741718528 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated35386389 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed10840047362896 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed84010482 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 130422799 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 130413143 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 130413143 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current252 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated212014065828864 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10866729231360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed82045627 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated223674392769024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed206745672829472 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed10866641801728 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated84010736 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed35386137 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed223674230899712 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed82045627 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated222880795060224 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11160651050496 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed212014048789504 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10866641801728 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated217585823034852 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated82045629 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated212014065828864 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed48624345 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated48624347 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed212513716814848 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0wc.2- _wandbruntime5481  _runtime5481j&M S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed222898736973824 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 212475936 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 212485608 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 212485608 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed82052270 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed84017556 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 130433336 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated84017800 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated212530948835328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated10841010639354 _runtime5481.105286151 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated10867608858112 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current242 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11161554215936 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 _step986 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated48628284 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed48628282 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated217603440680506 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10867521421312 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed82052270 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 212475936 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated223692503051264 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed223692343286272 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed212530923931648 % _timestamp1.7858007682819183e+09 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed212031215552512 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 130433336 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated212031232591872 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated82052272 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed217603337837910 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed10867521421312 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated82052272 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed212031215552512 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated35389516 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed35389274 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 trainer/global_step12349 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated222898841449984 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 30219960329RS$M OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 130433336 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated212031232591872 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated82052272 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed217603337837910 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed10867521421312 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated82052272 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed212031215552512 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated35389516 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed35389274 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 trainer/global_step12349 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated222898841449984 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed222898736973824 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 212475936 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 212485608 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 212485608 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed82052270 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed84017556 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 130433336 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated84017800 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated212530948835328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated10841010639354 _runtime5481.105286151 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated10867608858112 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current242 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11161554215936 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 _step986 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated48628284 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed48628282 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated217603440680506 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10867521421312 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed82052270 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 212475936 J;DeviceStatsMonitor.on_train_batch_enb훅d/active_bytes.all.peak 2940455936 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated223692503051264 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed223692343286272 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed212530923931648 % _timestamp1.7858007682819183e+09 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed212031215552512 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak862781442- _wandbruntime5481  _runtime5481ta _step987 /loss/FRt1/assignment_loss2.622952045396317 2loss/FBt2/assignment_loss4.05311321571844e-07 /loss/FRt1/detection_loss2.4500013494560884 /loss/FRt2/detection_loss2.2254056469315575 .loss/FBt2/detection_loss2.605864655226469 %loss/total_loss17.60182559038071 epoch0 0loss/FRt2/assignment_loss2.4583244508221034 2loss/SRqqt1/assignment_loss0.7285520119787475 2loss/SRqqt2/assignment_loss0.9054696901836583 3loss/FBt1/assignment_loss0.0014831846421124585 1loss/SRqqt1/detection_loss1.7100690445581148 1loss/SRqqt2/detection_loss0.8581936411980179 /loss/FBt1/detection_loss1.0355094646762029 trainer/global_step12349 % _timestamp1.7858007682838619e+09 _runtime 5481.10672954"  2loss/SRqqt1/assignment_loss0.7285520119787475 2loss/SRqqt2/assignment_loss0.9054696901836583 3loss/FBt1/assignment_loss0.0014831846421124585 1loss/SRqqt1/detection_loss1.7100690445581148 1loss/SRqqt2/detection_loss0.8581936411980179 /loss/FBt1/detection_loss1.0355094646762029 trainer/global_step12349 % _timestamp1.7858007682838619e+09 _runtime 5481.10672954 _step987 /loss/FRt1/assignment_loss2.622952045396317 2loss/FBt2/assignment_loss4.05311321571844e-07 /loss/FRt1/detection_loss2.4500013494560884 /loss/FRt2/detection_loss2.2254056469315575 .loss/FBt2/detection_loss2.605864655226469 %loss/total_loss17.60182559038071 epoch0 0loss/FRt2/assignment_loss2.4583244508221034YWFZ(:  U disk.drbd2118.in0 disk.drbd1817.in0 disk.drbd2041.in0 disk.drbd1289.in0 disk.drbd2118.out0 disk.drbd1356.in0 disk.drbd1356.out0$ memory_percent41.25160574913025 network.sent 1.1344553e+07 disk.sda.out 149.11328125 disk.drbd2111.in0 disk.drbd1817.out0 disk.rbd1.in0* proc.memory.availableMB38501.34765625! proc.memory.rssMB 11260.40625 disk.drbd1552.in0 disk.rbd0.in0 disk.sda1.out0 disk.drbd1828.out0 disk.drbd1289.out0 disk.drbd1931.in0 disk.drbd1285.out0 disk.drbd2078.in0 disk.drbd1938.out0 disk.drbd1510.out0 disk.dm-0.in 4.64453125 disk.sda2.in0* proc.memory.percent17.182016372680664 disk.drbd1931.out0 disk.drbd1285.in0 disk.drbd2078.out0 disk.drbd2111.out0 disk.drbd1510.in0 disk.drbd1584.out0$ disk./.usageGB578.3499526977539 disk.drbd1673.out0 disk.drbd1965.out0 disk.drbd1584.in0" disk.nvme0n1.out 260.529296875 cpu13.760182922621068 network.recv653290 disk.drbd2041.out0 disk.dm-0.out 260.529296875* disk./.usagePercent32.344356988824785 disk.rbd0.out0 disk.sda.in0 disk.nvme0n1.in 4.64453125 disk.rbd1.out 22.0908203125 disk.drbd1965.in0 proc.cpu.threads29 disk.drbd1828.in0 disk.sda1.in0 disk.drbd1226.out0 disk.drbd1550.out0 disk.drbd1938.in0 disk.drbd1673.in0 disk.drbd1226.in0 disk.sda2.out 149.11328125 disk.drbd1552.out0 disk.drbd1550.in0O4(: U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts117.732' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent39.244 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0:B2- _wandbruntime5502  _runtime5502C & lr-AdamW/pg10.0003195493402313103 & lr-AdamW/pg20.0003195493402313103 trainer/global_step12399 % _timestamp1.7858007898074517e+09 _runtime5502.630411027 _step988 & lr-AdamW/pg20.0003195493402313103 trainer/global_step12399 % _timestamp1.7858007898074517e+09 _runtime5502.630411027 _step988 & lr-AdamW/pg10.0003195493402313103 2- _wandbruntime5502  _runtime5502'1'O H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated212872403979264 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10910637937152 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed11205748213248 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed218466617271876 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated82377779 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 % _timestamp1.7858007898084111e+09 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 130941145 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated207582698484832 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10910725366784 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated84350528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed212872386939904 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 trainer/global_step12399 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated213374097558528 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11205885178880 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated82377779 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 130941145 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 130950801 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 _step989 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed35529086 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 _runtime5502.631643881 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed223783024877056 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated223783129346048 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated35529331 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed82377777 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current245 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed224579820868096 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated10884021629528 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed82377777 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated48821197 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated224579982737408 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated218466720114360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 213328580 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed84350281 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated223783129346048 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed223783024877056 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed212872386939904 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed10883935826404 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed10910637937152 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated212872403979264 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10910637937152 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed11205748213248 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed218466617271876 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated82377779 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 % _timestamp1.7858007898084111e+09 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 130941145 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated207582698484832 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10910725366784 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated84350528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed212872386939904 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 trainer/global_step12399 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated213374097558528 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11205885178880 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated82377779 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 130941145 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 130950801 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 _step989 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed35529086 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 _runtime5502.631643881 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed223783024877056 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated223783129346048 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated35529331 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed82377777 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current245 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed224579820868096 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated10884021629528 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed82377777 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640zh2- _wandbruntime5503  _runtime5503c aM XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed224597929140224 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 trainer/global_step12399 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current262 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed213391279771648 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current260 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 130961338 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 213345760 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11206649368576 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated212889570742272 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated223801175735808 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed223801071259648 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed82384420 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed207599421617792 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 _step990 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed212889553702912 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated218484337760014 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 213336088 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11206780035584 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated223801175735808 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 130961338 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed84357067 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 213336088 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current 24903680 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _runtime5503.031191289 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated82384422 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10911517556736 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed48825132 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated10911604993536 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated207599438657152 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed82384420 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed212889553702912 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated35532195 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed3553193513&M ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated207599438657152 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed82384420 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed212889553702912 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated35532195 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed35531935 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed224597929140224 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 trainer/global_step12399 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current262 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed213391279771648 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current260 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 130961338 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 213345760 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11206649368576 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated212889570742272 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated223801175735808 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed223801071259648 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed82384420 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed207599421617792 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 _step990 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed212889553702912 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated218484337760014 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 213336088 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11206780035584 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated223801175735808 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 130961338 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed84357067 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 213336088 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _runtime5503.031191289 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated82384422 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10911517556736 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed48825132 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated10911604993536 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251WR#2- _wandbruntime5503  _runtime5503od  /loss/FRt1/assignment_loss2.616790239201036 0loss/FRt2/assignment_loss2.3672158549813664 2loss/SRqqt1/assignment_loss0.6387385281753571 2loss/SRqqt2/assignment_loss0.9036037817131728 3loss/FBt1/assignment_loss0.0005030572943173834 2loss/FBt2/assignment_loss0.020513102412218487 1loss/SRqqt2/detection_loss0.8714560762194818 /loss/FBt1/detection_loss0.9155672415056162 epoch0 trainer/global_step12399 $ _timestamp1.785800790209537e+09 _runtime5503.032386574 /loss/FBt2/detection_loss1.5219228151545394 %loss/total_loss16.65698444790929 /loss/FRt1/detection_loss2.5038125506320665 1loss/SRqqt1/detection_loss1.9142342951963656 _step991 /loss/FRt2/detection_loss2.3826269054237534gw /loss/FRt1/assignment_loss2.616790239201036 0loss/FRt2/assignment_loss2.3672158549813664 2loss/SRqqt1/assignment_loss0.6387385281753571 2loss/SRqqt2/assignment_loss0.9036037817131728 3loss/FBt1/assignment_loss0.0005030572943173834 2loss/FBt2/assignment_loss0.020513102412218487 1loss/SRqqt2/detection_loss0.8714560762194818 /loss/FBt1/detection_loss0.9155672415056162 epoch0 trainer/global_step12399 $ _timestamp1.785800790209537e+09 _runtime5503.032386574 /loss/FBt2/detection_loss1.5219228151545394 %loss/total_loss16.65698444790929 /loss/FRt1/detection_loss2.5038125506320665 1loss/SRqqt1/detection_loss1.9142342951963656 _step991 /loss/FRt2/detection_loss2.3826269054237534б U(:  U disk.drbd1550.in0$ memory_percent41.25239849090576* disk./.usagePercent32.344364882324896 disk.drbd1673.out0 disk.drbd1289.out0 disk.rbd0.in0 disk.drbd2078.in0 network.sent 1.1389539e+07 disk.rbd1.in0 disk.drbd1828.in0 disk.drbd1289.in0 disk.dm-0.out 260.76171875 disk.drbd1817.in0 disk.rbd0.out0 disk.sda.in0 disk.sda1.out0* proc.memory.percent17.182016372680664! disk.nvme0n1.out 260.76171875 cpu13.759571697416927 disk.nvme0n1.in 4.64453125 disk.drbd1356.out0 disk.drbd2041.out0 disk.drbd1285.out0 disk.drbd1356.in0 disk.drbd2041.in0 network.recv654764 disk.sda.out 149.31640625 disk.sda2.in0 disk.sda2.out 149.31640625 disk.drbd1552.in0 disk.drbd1938.in0 disk.drbd1584.out0 disk.drbd2118.out0 disk.drbd1965.out0 disk.drbd1226.in0 disk.drbd2111.out0 disk.drbd1938.out0 disk.drbd1226.out0 disk.drbd2118.in0 disk.drbd1673.in0 disk.drbd1552.out0 disk.drbd1510.out0 disk.drbd1584.in0 disk.rbd1.out 22.123046875 disk.drbd1828.out0 disk.drbd1550.out0( proc.memory.availableMB 38500.828125 proc.cpu.threads29 disk.drbd2111.in0 disk.drbd1931.in0 disk.dm-0.in 4.64453125$ disk./.usageGB578.3500938415527 disk.drbd1817.out0 disk.drbd1931.out0 disk.sda1.in0 disk.drbd1285.in0 disk.drbd1510.in0 disk.drbd2078.out0! proc.memory.rssMB 11260.40625 disk.drbd1965.in0<3jE(: U gpu.0.gpu6.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts95.749' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent31.91633333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0E_(: ڷU gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts129.741' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent43.247 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0%+gR(:  ߆V disk.drbd1673.out0 disk.drbd1550.out0! proc.memory.rssMB 11260.40625 disk.sda2.out 149.6640625$ memory_percent41.24863147735596 disk.rbd1.out 22.216796875) disk./.usagePercent32.34433821509479 disk.drbd1828.out0 disk.dm-0.out 260.99609375 disk.drbd2078.in0 disk.nvme0n1.in 4.64453125 disk.drbd1289.out0 disk.sda.in0 disk.sda1.in0 disk.drbd1510.in0 disk.drbd1965.in0 disk.drbd1289.in0 disk.drbd1938.out0 network.sent 1.1434753e+07$ disk./.usageGB578.3496170043945 disk.drbd1226.out0 disk.drbd2078.out0 disk.drbd1965.out0 disk.drbd1552.in0 disk.rbd0.in0 disk.rbd0.out0 disk.drbd2111.in0 disk.drbd1828.in0 disk.drbd1938.in0 cpu13.759096964657898 disk.drbd1584.out0! disk.nvme0n1.out 260.99609375 disk.drbd1550.in0 disk.sda.out 149.6640625 disk.drbd1931.in0 disk.drbd2041.out0 disk.dm-0.in 4.64453125 disk.drbd2118.in0 disk.drbd1356.out0 disk.drbd1552.out0 disk.drbd1510.out0 network.recv656497 disk.drbd1931.out0 disk.drbd1356.in0 disk.drbd2111.out0 disk.drbd1817.out0 disk.drbd1285.in0( proc.memory.availableMB 38503.296875* proc.memory.percent17.182016372680664 proc.cpu.threads29 disk.sda1.out0 disk.sda2.in0 disk.drbd2041.in0 disk.drbd1226.in0 disk.drbd2118.out0 disk.drbd1817.in0 disk.rbd1.in0 disk.drbd1584.in0 disk.drbd1673.in0 disk.drbd1285.out07a2- _wandbruntime5524  _runtime5524 _runtime5524.515222758 _step992 & lr-AdamW/pg10.0003208379495555756 & lr-AdamW/pg20.0003208379495555756 trainer/global_step12449 # _timestamp1.78580081169219e+09` & lr-AdamW/pg10.0003208379495555756 & lr-AdamW/pg20.0003208379495555756 trainer/global_step12449 # _timestamp1.78580081169219e+09 _runtime5524.515222758 _step992n2- _wandbruntime5524  _runtime55246'O G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated84692429 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed10927824289912 _runtime5524.516651756 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10954634072576 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11251077008384 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated10954721502208 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed82709927 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 214188732 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed49018045 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 214179074 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated224685463631872 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed224685359162880 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed11250942139904 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated219347617193868 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated213730742129664 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _step993 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current278 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated10927910093036 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated208419707100832 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 131469147 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated82709929 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 131469147 % _timestamp1.7858008116932898e+09 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed213730725090304 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10954721502208 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated213730742129664 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 131478803 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed224685359162880 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed10954634072576 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated225485530406912 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed219347514351384 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed225485370634752 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed214234428494848 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated82709929 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed213730725090304 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated214234453398528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated49018047 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated35674382 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed35674106 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed84692151 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed208419690061472 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed82709927 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130؃4O JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed224685359162880 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed10954634072576 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated225485530406912 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed219347514351384 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed225485370634752 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed214234428494848 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated82709929 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed213730725090304 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated214234453398528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated49018047 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated35674382 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed35674106 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed84692151 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed208419690061472 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed82709927 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 LDDeviceStatsMonitor.oncNS_train_batch_start/segment.large_pool.allocated130 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated84692429 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed10927824289912 _runtime5524.516651756 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10954634072576 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11251077008384 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated10954721502208 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed82709927 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 214188732 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed49018045 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 214179074 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated224685463631872 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed224685359162880 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed11250942139904 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated219347617193868 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated213730742129664 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _step993 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current278 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated10927910093036 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated208419707100832 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 131469147 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated82709929 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 131469147 % _timestamp1.7858008116932898e+09 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed213730725090304 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10954721502208 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated213730742129664 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 131478803 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 J6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated84699151 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated35677167 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime5524.883282643 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed10955513692160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed219365131996926 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed213747891853312 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed213747891853312 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed84698893 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 214205912 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 131479670 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated82716572 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated10955601128960 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated214251660515328 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed225503481847808 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 131489340 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current258 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated219365234839522 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated224703510021632 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10955513692160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated225503637418496 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated208436447273152 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated10928787566370 $ _timestamp1.785800812059847e+09 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 214196240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed49021982 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current256 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated213747908892672 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed214251635611648 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed224703405545472 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated213747908892672 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 _step994 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated82716572 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 131479670 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 &M REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated82716572 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 131479670 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11251976903168 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed82716570 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated224703510021632 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated84699151 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated35677167 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime5524.883282643 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed10955513692160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed219365131996926 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed213747891853312 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed213747891853312 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed84698893 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 214205912 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 131479670 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated82716572 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated10955601128960 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated214251660515328 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed225503481847808 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 131489340 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current258 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated219365234839522 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated224703510021632 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10955513692160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated225503637418496 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated208436447273152 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated10928787566370 $ _timestamp1.785800812059847e+09 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 214196240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed49021982 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current256 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated213747908892672 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed214251635611648 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed224703405545472 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated213747908892672 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 _step994^`2- _wandbruntime5524  _runtime5524 4loss/FBt1/assignment_loss0.00035432658108618397 1loss/SRqqt1/detection_loss1.8396311909891665 0loss/SRqqt2/detection_loss0.987015483709356 /loss/FBt1/detection_loss1.0112439754108589 /loss/FRt1/assignment_loss2.495588380162434 0loss/FRt2/assignment_loss2.3074152322297685 2loss/SRqqt1/assignment_loss0.8851479421913003 2loss/FBt2/assignment_loss0.001029979543421756 .loss/FBt2/detection_loss1.342973923479969 trainer/global_step12449 /loss/FRt1/detection_loss2.4469161741836096 -loss/FRt2/detection_loss2.19353940865297 % _timestamp1.7858008120616817e+09 _step995 2loss/SRqqt2/assignment_loss0.7921022118171611 &loss/total_loss16.302958228951102 epoch0 _runtime5524.884534386Y  /loss/FRt1/detection_loss2.4469161741836096 -loss/FRt2/detection_loss2.19353940865297 % _timestamp1.7858008120616817e+09 _step995 2loss/SRqqt2/assignment_loss0.7921022118171611 &loss/total_loss16.302958228951102 epoch0 _runtime5524.884534386 4loss/FBt1/assignment_loss0.00035432658108618397 1loss/SRqqt1/detection_loss1.8396311909891665 0loss/SRqqt2/detection_loss0.987015483709356 /loss/FBt1/detection_loss1.0112439754108589 /loss/FRt1/assignment_loss2.495588380162434 0loss/FRt2/assignment_loss2.3074152322297685 2loss/SRqqt1/assignment_loss0.8851479421913003 2loss/FBt2/assignment_loss0.001029979543421756 .loss/FBt2/detection_loss1.342973923479969 trainer/global_step12449:\(:  ˎU disk.drbd1285.in0 disk.drbd2118.in0 disk.nvme0n1.in 4.64453125$ disk./.usageGB578.3495597839355 disk.drbd1931.out0 disk.drbd1828.in0! disk.dm-0.out261.26416015625 disk.drbd2111.out0 disk.drbd1817.in0 disk.drbd1965.out0 disk.drbd1550.in0 disk.drbd1289.out0$ disk.nvme0n1.out261.26416015625 disk.drbd1673.out0 disk.sda.in0 disk.drbd2078.out0 disk.drbd2118.out0! proc.memory.rssMB 11260.40625 disk.drbd1226.out0 disk.drbd1584.in0 disk.drbd1965.in0* proc.memory.percent17.182016372680664 disk.drbd1673.in0 disk.rbd0.out0 disk.drbd1550.out0 disk.drbd1356.in0 proc.cpu.threads29 disk.drbd1289.in0 disk.drbd1938.out0 disk.rbd0.in0 disk.sda.out 150.07421875 disk.sda1.in0 disk.drbd1285.out0 disk.drbd1226.in0% memory_percent41.247910261154175) disk./.usagePercent32.34433501502718 cpu13.75855984414242 network.recv657422 disk.sda2.out 150.07421875 disk.drbd2041.in0 disk.drbd1938.in0 disk.sda1.out0 disk.drbd1828.out0 disk.dm-0.in 4.64453125 disk.drbd1584.out0 disk.drbd2111.in0 disk.drbd1931.in0 disk.drbd1510.in0 disk.drbd1510.out0 disk.drbd1356.out0 disk.sda2.in0 disk.rbd1.out 22.279296875* proc.memory.availableMB38503.76953125 network.sent 1.1437896e+07 disk.drbd1552.in0 disk.drbd1552.out0 disk.drbd2078.in0 disk.drbd1817.out0 disk.rbd1.in0 disk.drbd2041.out0((: V gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts131.908' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent43.969333333333324 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0"L2- _wandbruntime5546  _runtime5546C _step996 & lr-AdamW/pg10.0003221265588798409 & lr-AdamW/pg20.0003221265588798409 trainer/global_step12499 % _timestamp1.7858008339166248e+09 _runtime5546.739606365Ƀ _runtime5546.739606365 _step996 & lr-AdamW/pg10.0003221265588798409 & lr-AdamW/pg20.0003221265588798409 trainer/global_step12499 % _timestamp1.7858008339166248e+092- _wandbruntime5546  _runtime5546O ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated226391056721408 trainer/global_step12499 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 131997149 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated10971798556544 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated83042079 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated83042079 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed220228411430892 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed10998630208000 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed214589063240704 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 215048884 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed83042077 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed226390903240704 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated215094809238528 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10998717637632 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed35818358 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed10971712753420 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 131997149 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated214589080280064 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 _step997 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed209256698677472 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated10998717637632 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed225587693448704 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed225587693448704 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10998630208000 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current233 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed215094784334848 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed49214895 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 215048884 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed83042077 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed226390903240704 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated215094809238528 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated10998717637632 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed35818358 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed10971712753420 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 131997149 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated214589080280064 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 _step997 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed209256698677472 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated10998717637632 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed225587693448704 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed225587693448704 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed10998630208000 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current233 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed215094784334848 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed49214895 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 J&M F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 215066064 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed214606230003712 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10999509827584 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated10999597264384 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 141152768 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed226409005106688 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated35821400 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated215112016355328 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed215111991451648 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 132007672 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 132017342 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 _step998 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed209273438849792 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated83048722 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed220246029076434 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current257 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated83048722 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed225605739831296 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed49218832 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated225605844307456 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 132017342 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 trainer/global_step12499 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed10972590226642 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated226409171163136 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated220246131919030 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated214606247043072 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed35821145 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed83048720 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11297013655040 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 $ _timestamp1.785800834297185e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 215066064 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime5547.120640535 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated225605844307456 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 215056392 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 132007672 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 166056448 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed214606230003712 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed85039977 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated85040234 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 215056392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated49218834 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated214606247043072 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11297154807808qvM OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 132017342 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 trainer/global_step12499 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed10972590226642 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated226409171163136 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated220246131919030 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated214606247043072 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed35821145 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed83048720 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11297013655040 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 $ _timestamp1.785800834297185e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 215066064 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime5547.120640535 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated225605844307456 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 215056392 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 132007672 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 166056448 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed214606230003712 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed85039977 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated85040234 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 215056392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated49218834 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated214606247043072 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 UGDeviceStatsMonitor.on_train_batch_end/reser >ved_bytes.small_pool.current 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11297154807808 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 215066064 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed214606230003712 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed10999509827584 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated10999597264384 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 141152768 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed226409005106688 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated35821400 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated215112016355328 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed215111991451648 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 132007672 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 132017342 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 _step998 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed209273438849792 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated83048722 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed220246029076434 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current257 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated83048722 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed225605739831296 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed49218832 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated225605844307456 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121tZA32- _wandbruntime5547  _runtime55471  4loss/FBt2/assignment_loss3.6159431502369443e-06 /loss/FRt2/detection_loss2.2938450424067947 1loss/SRqqt2/detection_loss0.9719245175430034 &loss/total_loss16.912172135285324 trainer/global_step12499 % _timestamp1.7858008342990534e+09 _step999 epoch0 _runtime5547.121956523 0loss/FRt2/assignment_loss2.2684012026671905 1loss/SRqqt1/assignment_loss1.074494700277732 .loss/FRt1/detection_loss2.855111134510774 /loss/FBt1/detection_loss1.0031921902502126 /loss/FBt2/detection_loss1.1323054110010464 0loss/FRt1/assignment_loss2.6653830931736873 2loss/SRqqt2/assignment_loss0.9733606639331103 3loss/FBt1/assignment_loss0.0005544432915483078 1loss/SRqqt1/detection_loss1.6735961202870715  0loss/FRt1/assignment_loss2.6653830931736873 2loss/SRqqt2/assignment_loss0.9733606639331103 3loss/FBt1/assignment_loss0.0005544432915483078 1loss/SRqqt1/detection_loss1.6735961202870715 4loss/FBt2/assignment_loss3.6159431502369443e-06 /loss/FRt2/detection_loss2.2938450424067947 1loss/SRqqt2/detection_loss0.9719245175430034 &loss/total_loss16.912172135285324 trainer/global_step12499 % _timestamp1.7858008342990534e+09 _step999 epoch0 _runtime5547.121956523 0loss/FRt2/assignment_loss2.2684012026671905 1loss/SRqqt1/assignment_loss1.074494700277732 .loss/FRt1/detection_loss2.855111134510774 /loss/FBt1/detection_loss1.0031921902502126 /loss/FBt2/detection_loss1.1323054110010464蚩(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts130.192' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent43.397333333333336 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors00׊X(:  U network.sent 1.148285e+07 disk.drbd1289.in0 disk.drbd1584.out0 disk.drbd1965.out0 disk.sda.in0 disk.drbd2041.out0 disk.sda2.out 150.37109375( proc.memory.availableMB 38503.203125 disk.drbd1828.in0 disk.drbd1828.out0 disk.drbd1510.out0 disk.drbd1817.in0 disk.drbd1817.out0 disk.drbd1285.out0 disk.drbd1965.in0 proc.cpu.threads29$ memory_percent41.24877452850342 disk.drbd1552.out0 disk.drbd1938.in0 disk.drbd1510.in0 disk.drbd1226.in0 disk.drbd1931.in0 disk.drbd1931.out0 disk.drbd2041.in0 disk.drbd1550.out0 disk.rbd0.in0 disk.drbd1226.out0 disk.drbd1673.out0 disk.dm-0.in 4.64453125 disk.sda1.out0 disk.drbd2118.out0 disk.drbd2111.out0 disk.drbd1673.in0 disk.drbd2118.in0 disk.nvme0n1.in 4.64453125 disk.rbd1.out 22.279296875 disk.drbd2078.out0 network.recv658936 disk.drbd1552.in0! proc.memory.rssMB 11260.40625 disk.rbd1.in0 disk.rbd0.out0 disk.sda.out 150.37109375! disk.dm-0.out261.45947265625$ disk.nvme0n1.out261.45947265625 cpu13.75813746122294) disk./.usagePercent32.34433949512183 disk.drbd1550.in0 disk.drbd2078.in0 disk.drbd1584.in0 disk.drbd1356.out0 disk.drbd1289.out0 disk.sda1.in0 disk.drbd1356.in0 disk.drbd2111.in0$ disk./.usageGB578.3496398925781 disk.drbd1938.out0 disk.drbd1285.in0 disk.sda2.in0* proc.memory.percent17.182016372680664P(: U gpu.0.gpu0.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts95.984' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.994666666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0N'pW(:  U disk.drbd2078.in0 disk.drbd2078.out0 disk.drbd1673.out0 disk.drbd1550.out0 disk.dm-0.in 4.64453125 disk.sda2.in0 disk.drbd2111.in0$ memory_percent41.24915599822998 disk.drbd1673.in0 disk.drbd1828.in0 disk.sda.out 150.609375! disk.dm-0.out261.69873046875 disk.drbd2111.out0 disk.drbd1817.in0 disk.drbd1965.out0$ disk./.usageGB578.3496589660645 disk.sda1.in0 disk.drbd1285.in0 disk.drbd2118.in0 disk.drbd1550.in0 disk.sda.in0 disk.drbd1285.out0 disk.drbd1584.in0 disk.drbd2118.out0 proc.cpu.threads29 disk.drbd1931.out0 disk.drbd1938.in0 disk.drbd1510.in0 disk.drbd1356.out0 disk.rbd1.out 22.3115234375 cpu13.757650207058171 network.sent 1.1527919e+07 disk.drbd1552.in0( proc.memory.availableMB 38502.953125 disk.rbd0.out0 disk.drbd1226.out0 disk.drbd1828.out0 disk.rbd0.in0 disk.nvme0n1.in 4.64453125 disk.drbd1965.in0 disk.drbd1552.out0 disk.drbd1938.out0 disk.drbd1510.out0 disk.sda1.out0 disk.drbd2041.out0 disk.drbd1289.in0 disk.drbd1289.out0! proc.memory.rssMB 11260.40625 disk.drbd1931.in0 disk.drbd1584.out0 network.recv660556) disk./.usagePercent32.34434056181104 disk.drbd2041.in0 disk.drbd1356.in0 disk.drbd1817.out0* proc.memory.percent17.182016372680664 disk.drbd1226.in0$ disk.nvme0n1.out261.69873046875 disk.sda2.out 150.609375 disk.rbd1.in02- _wandbruntime5568  _runtime5568' & lr-AdamW/pg10.0003234151682041062 & lr-AdamW/pg20.0003234151682041062 trainer/global_step12549 % _timestamp1.7858008554866571e+09 _runtime5568.309627659 _step1000+I7 & lr-AdamW/pg10.0003234151682041062 & lr-AdamW/pg20.0003234151682041062 trainer/global_step12549 % _timestamp1.7858008554866571e+09 _runtime5568.309627659 _step1000Bam2- _wandbruntime5568  _runtime5568޴'O JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed83374227 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed215447401391104 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11042626343424 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 132534807 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11341395864064 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11015687020052 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed221109308510400 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed210093707293472 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current270 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed85373792 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 _step1001 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed226490027734528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed226490027734528 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed227296407461888 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 _runtime5568.310850221 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed83374227 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated83374229 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed215447401391104 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed49411745 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed11015601216928 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11042626343424 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated210093724332832 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated226490132203520 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 215909036 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated83374229 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed215955140174848 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed83374227 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed215447401391104 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11042626343424 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 132534807 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11341395864064 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11015687020052 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed221109308510400 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed210093707293472 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current270 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed85373792 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 _step1001 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed226490027734528 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed226490027734528 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed227296407461888 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 _runtime5568.310850221 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed83374227 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated83374229 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed215447401391104 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed49411745 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed11015601216928 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11042626343424 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated210093724332832 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated226490132203520 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 215909036 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated83374229 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed215955140174848 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed35964813 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11342169662976 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated227314672525312 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed221126926155942 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated85380750 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated35965066 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11016564493386 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 215926216 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current255 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 132545344 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated83380872 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 215926216 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed83380870 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated49415684 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11016478690150 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated11043593399808 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed11043505963008 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11043505963008 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed215972347291648 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated210110464505152 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed85380495 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 132545344 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11342300329984 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated226508178593280 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 132535674 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0&M L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated226508178593280 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 132535674 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed226508074117120 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed83380870 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed227314516954624 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current253 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 _step1002 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated215464585193472 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated215464585193472 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated11043593399808 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 215916544 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed215464568154112 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed35964813 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11342169662976 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated227314672525312 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed221126926155942 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated85380750 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated35965066 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11016564493386 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 215926216 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current255 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 132545344 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated83380872 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 215926216 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed83380870 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated49415684 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11016478690150 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated11043593399808 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed11043505963008 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11043505963008 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed215972347291648 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated210110464505152 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed85380495 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 132545344 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11342300329984 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121MNt2- _wandbruntime5568  _runtime5568  .loss/FRt2/detection_loss2.447728030477898 1loss/SRqqt1/detection_loss2.0078771772483983 /loss/FBt1/detection_loss1.1628961911260642 trainer/global_step12549 _step1003 2loss/SRqqt1/assignment_loss0.8865261543985634 2loss/SRqqt2/assignment_loss0.8004197219426346 /loss/FBt2/detection_loss1.1396157967392355 &loss/total_loss16.769280300825386 epoch0 _runtime5568.680091224 /loss/FRt1/detection_loss2.5174813472562367 1loss/SRqqt2/detection_loss1.0522186374561726 % _timestamp1.7858008558572595e+09 0loss/FRt1/assignment_loss2.6259849607944488 4loss/FBt1/assignment_loss1.4690645065396733e-06 /loss/FRt2/assignment_loss2.128451688380181 3loss/FBt2/assignment_loss7.912594104869047e-05F 0loss/FRt1/assignment_loss2.6259849607944488 4loss/FBt1/assignment_loss1.4690645065396733e-06 /loss/FRt2/assignment_loss2.128451688380181 3loss/FBt2/assignment_loss7.912594104869047e-05 .loss/FRt2/detection_loss2.447728030477898 1loss/SRqqt1/detection_loss2.0078771772483983 /loss/FBt1/detection_loss1.1628961911260642 trainer/global_step12549 _step1003 2loss/SRqqt1/assignment_loss0.8865261543985634 2loss/SRqqt2/assignment_loss0.8004197219426346 /loss/FBt2/detection_loss1.1396157967392355 &loss/total_loss16.769280300825386 epoch0 _runtime5568.680091224 /loss/FRt1/detection_loss2.5174813472562367 1loss/SRqqt2/detection_loss1.0522186374561726 % _timestamp1.7858008558572595e+09i˨(: U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts101.045' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent33.681666666666665 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0@X\(:  ǃU disk.sda1.in0 disk.drbd1584.out0 disk.drbd2118.out0* proc.memory.percent17.182016372680664 disk.drbd2041.out0 disk.drbd1226.in0 disk.drbd2078.in0 disk.drbd2111.in0 disk.rbd1.in0! proc.memory.rssMB 11260.40625 cpu13.757230664835177$ disk.nvme0n1.out261.93310546875 disk.drbd1356.in0 disk.drbd1285.out0 disk.drbd2118.in0 disk.drbd1356.out0 disk.drbd1931.out0 disk.drbd1550.out0 disk.drbd1828.in0 disk.dm-0.in 4.64453125 disk.drbd1552.in0$ disk./.usageGB578.3496208190918 disk.sda.in0 disk.drbd2078.out0* proc.memory.availableMB38503.16796875 disk.drbd1285.in0) disk./.usagePercent32.34433842843263 disk.drbd1817.in0 disk.drbd1510.in0 disk.nvme0n1.in 4.64453125 disk.sda2.out 150.9140625 disk.drbd1673.in0 disk.drbd1552.out0 disk.drbd1289.in0 disk.drbd1289.out0% memory_percent41.248828172683716 disk.drbd1226.out0 disk.drbd1584.in0 disk.drbd1938.out0 disk.sda.out 150.9140625 disk.drbd1673.out0 disk.drbd1828.out0 disk.drbd1550.in0 disk.drbd2111.out0 disk.drbd1965.in0 disk.rbd0.out0! disk.dm-0.out261.93310546875 disk.sda2.in0 network.sent 1.1530938e+07 disk.drbd1938.in0 disk.rbd0.in0 disk.sda1.out0 disk.drbd1510.out0 disk.drbd1965.out0 network.recv661575 disk.drbd1817.out0 disk.drbd1931.in0 proc.cpu.threads29 disk.rbd1.out 22.4052734375 disk.drbd2041.in07h2- _wandbruntime5590  _runtime5590dT%k _runtime5590.440593049 _step1004 & lr-AdamW/pg10.0003247037775283715 & lr-AdamW/pg20.0003247037775283715 trainer/global_step12599 % _timestamp1.7858008776176124e+09 & lr-AdamW/pg10.0003247037775283715 & lr-AdamW/pg20.0003247037775283715 trainer/global_step12599 % _timestamp1.7858008776176124e+09 _runtime5590.440593049 _step1004$2- _wandbruntime5590  _runtime5590OTh#O ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated228202125056512 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 133062809 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 216759530 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed216305739541504 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated49608597 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed228201965284352 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed36104793 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed227392362020352 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed11059489680436 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated83706379 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 _runtime5590.445210822 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed216305739541504 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current257 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed210930715909472 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed83706377 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated85713645 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11386604137984 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 _step1005 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed227392362020352 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed49608595 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed216815496014848 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed11386469269504 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed221990205589908 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11059575483560 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated210930732948832 $ _timestamp1.785800877618553e+09 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11086622478848 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 133053153 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 133053153 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated216305756580864 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed85713388 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated227392466489344 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed227392362020352 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed11059489680436 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated83706379 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 _runtime5590.445210822 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed216305739541504 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current257 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed210930715909472 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed83706377 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated85713645 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11386604137984 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 _step1005 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed227392362020352 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed49608595 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed216815496014848 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed11386469269504 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed221990205589908 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11059575483560 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated210930732948832 $ _timestamp1.785800877618553e+09 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11086622478848 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 133053153 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 133053153 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated216305756580864 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed85713388 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated227392466489344 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 JDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 % _timestamp1.7858008780430534e+09 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed83713020 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed85720129 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed227410408402944 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated11087589535232 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 133063676 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated210947473121152 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 216786368 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed49612532 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated228220231560192 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed83713020 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated222007926078046 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated83713022 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current241 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 _step1006 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 216776696 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed11087502098432 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 216786368 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated36107838 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated216832728035328 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current243 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11060367153658 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11087502098432 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 trainer/global_step12599 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 133063676 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 133073346 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed228220073892352 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed222007823235450 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 _runtime5590.866635494 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated216322923343872 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 216776696 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated85720372 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11060452956894D&M WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated222007926078046 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated83713022 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current241 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 _step1006 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 216776696 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed11087502098432 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 216786368 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated36107838 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated216832728035328 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current243 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11060367153658 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11087502098432 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 trainer/global_step12599 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 133063676 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 133073346 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed228220073892352 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed222007823235450 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 _runtime5590.866635494 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated216322923343872 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 216776696 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated85720372 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11060452956894 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 % _timestamp1.7858008780430534e+09 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed83713020 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed85720129 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed227410408402944 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated11087589535232 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 133063676 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated210947473121152 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 216786368 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed49612532 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated228220231560192 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed83713020 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291]2- _wandbruntime5590  _runtime5590  0loss/FRt1/assignment_loss2.4176387017773044 3loss/FBt1/assignment_loss6.622737518884808e-09 .loss/FRt2/detection_loss2.355211717795722 1loss/SRqqt1/assignment_loss0.793878084644242 /loss/FRt1/detection_loss2.3948688407739005 /loss/FBt1/detection_loss0.6811595405710654 &loss/total_loss15.667420647024553 epoch0 _step1007 0loss/FRt2/assignment_loss1.9587801725049563 3loss/FBt2/assignment_loss9.636060767093113e-07 1loss/SRqqt2/detection_loss1.0543882524190369 /loss/FBt2/detection_loss1.6782547644494723 trainer/global_step12599 2loss/SRqqt2/assignment_loss0.8319756221892529 1loss/SRqqt1/detection_loss1.5012639796707865 $ _timestamp1.785800878044768e+09 _runtime5590.867720096z 1loss/SRqqt2/detection_loss1.0543882524190369 /loss/FBt2/detection_loss1.6782547644494723 trainer/global_step12599 2loss/SRqqt2/assignment_loss0.8319756221892529 1loss/SRqqt1/detection_loss1.5012639796707865 $ _timestamp1.785800878044768e+09 _runtime5590.867720096 0loss/FRt1/assignment_loss2.4176387017773044 3loss/FBt1/assignment_loss6.622737518884808e-09 .loss/FRt2/detection_loss2.355211717795722 1loss/SRqqt1/assignment_loss0.793878084644242 /loss/FRt1/detection_loss2.3948688407739005 /loss/FBt1/detection_loss0.6811595405710654 &loss/total_loss15.66і7420647024553 epoch0 _step1007 0loss/FRt2/assignment_loss1.9587801725049563 3loss/FBt2/assignment_loss9.636060767093113e-07 !(: U gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts127.551' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent42.516999999999996 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0E{P(:  U network.recv663208 disk.drbd2118.out0 disk.drbd1285.in0 disk.dm-0.in 4.64453125 disk.drbd1584.in0 disk.drbd1673.in0 disk.drbd1510.out0 disk.drbd1356.in0 disk.drbd1552.out0 disk.drbd1550.out0 disk.nvme0n1.out 262.1484375$ memory_percent41.24927520751953 disk.drbd2078.out0 disk.drbd1931.in0 disk.drbd2111.out0 disk.drbd1965.in0 disk.drbd1817.out0 disk.drbd1931.out0 disk.drbd2041.out0 disk.drbd1289.out0 disk.drbd1510.in0 disk.dm-0.out 262.1484375! proc.memory.rssMB 11260.40625 disk.drbd1817.in0 disk.drbd2041.in0 disk.drbd2118.in0 disk.drbd1356.out0 disk.drbd1673.out0 disk.drbd1550.in0 disk.sda.out 151.19140625 disk.sda1.out0 disk.sda2.in0 disk.sda2.out 151.19140625 disk.rbd0.in0 disk.rbd0.out0 network.sent 1.1576021e+07 disk.drbd2111.in0 disk.drbd1938.out0 proc.cpu.threads29$ disk./.usageGB578.3496208190918 disk.drbd1965.out0 disk.drbd1552.in0 disk.drbd1226.in0 disk.drbd1226.out0 disk.drbd2078.in0 disk.drbd1938.in0 disk.sda.in0 disk.rbd1.in0 disk.rbd1.out 22.4677734375 disk.drbd1289.in0 disk.sda1.in0% proc.memory.availableMB 38502.875 disk.drbd1828.in0 disk.drbd1828.out0* proc.memory.percent17.182016372680664) disk./.usagePercent32.34433842843263 disk.drbd1584.out0 disk.nvme0n1.in 4.64453125 disk.drbd1285.out0 cpu13.756724938511816 E(: U gpu.0.gpu41.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts126.202' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.06733333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors05HV(:  U disk.drbd1226.in0 disk.rbd0.in0 disk.rbd0.out0 disk.drbd1510.in0 disk.drbd2041.out0 disk.drbd1510.out0( proc.memory.availableMB 38502.609375 disk.drbd2118.in0 disk.drbd1584.in0 disk.drbd1584.out0 disk.nvme0n1.in 4.64453125 disk.drbd1356.in0 disk.drbd2041.in0 disk.drbd1828.in0 disk.drbd1285.in0 network.recv664770 disk.drbd1938.out0 disk.drbd1226.out0 disk.drbd2078.in0 disk.drbd2111.out0 disk.sda.in0 disk.sda1.out0! proc.memory.rssMB 11260.40625 disk.drbd2118.out0 disk.sda2.in0 disk.sda.out 151.56640625 disk.rbd1.out 22.4677734375 disk.drbd1965.out0 disk.drbd1550.out0 disk.sda2.out 151.56640625 cpu13.756244504392667% memory_percent41.249680519104004 network.sent 1.1620976e+07 disk.drbd2078.out0 disk.drbd1289.in0 disk.drbd1285.out0 proc.cpu.threads29 disk.dm-0.in 4.64453125! disk.nvme0n1.out 262.43359375 disk.drbd1673.out0 disk.drbd2111.in0 disk.drbd1673.in0 disk.drbd1931.out0 disk.drbd1552.out0 disk.drbd1817.out0 disk.drbd1828.out0 disk.sda1.in0 disk.drbd1817.in0 disk.drbd1965.in0 disk.drbd1938.in0 disk.dm-0.out 262.43359375) disk./.usagePercent32.34433842843263 disk.rbd1.in0 disk.drbd1931.in0 disk.drbd1552.in0 disk.drbd1289.out0$ disk./.usageGB578.3496208190918 disk.drbd1356.out0 disk.drbd1550.in0* proc.memory.percent17.182016372680664B2- _wandbruntime5612  _runtime5612E0 ' lr-AdamW/pg20.00032599238685263683 trainer/global_step12649 % _timestamp1.7858008995821865e+09 _runtime5612.405167588 _step1008 ' lr-AdamW/pg10.00032599238685263683 ײ % _timestamp1.7858008995821865e+09 _runtime5612.405167588 _step1008 ' lr-AdamW/pg10.00032599238685263683 ' lr-AdamW/pg20.00032599238685263683 trainer/global_step12649N2- _wandbruntime5612  _runtime5612% 'O A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated11130706043904 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11103463947068 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated84038529 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed84038527 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed217164077691904 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed228294696306176 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed36248184 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated211767741564832 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed228294696306176 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 133590811 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 _step1009 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated36248449 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated217164094731264 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed84038527 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current267 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated11130706043904 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed86053629 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated49805447 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated222871205511900 % _timestamp1.7858008995831375e+09 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated228294800775168 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated217164094731264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed229107528946688 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed11103378143944 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 _runtime5612.409674075 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 217619682 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed217164077691904 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 133590811 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated86053896 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11431814057472 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11130618614272 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated229107690816000 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed217675851854848 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed222871102669416 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed211767724525472 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed49805445 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated228294800775168 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11130618614272 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current265 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated217675876758528 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200b'O @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated211767741564832 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed228294696306176 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 133590811 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 _step1009 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated36248449 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated217164094731264 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed84038527 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current267 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated11130706043904 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed86053629 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated49805447 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated222871205511900 % _timestamp1.7858008995831375e+09 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated228294800775168 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated217164094731264 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed229107528946688 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed11103378143944 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 _runtime5612.409674075 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 217619682 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed217164077691904 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 133590811 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated86053896 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11431814057472 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11130618614272 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated229107690816000 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed217675851854848 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed222871102669416 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed211767724525472 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed49805445 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated228294800775168 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11130618614272 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current265 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated217675876758528 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated11130706043904 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11103463947068 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated84038529 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed84038527 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed217164077691904 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed228294696306176 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed36248184{׭2- _wandbruntime5612  _runtime5612>|<M REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated86060606 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed49809382 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated228312847164928 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 217636848 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 133601348 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11104255617166 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed86060360 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed222888720314958 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11104341420402 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11432588364288 $ _timestamp1.785800899952075e+09 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated217181261494272 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated217181261494272 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 _runtime5612.775472845 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 217636848 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 217646520 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed36250978 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated84045172 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated217693083875328 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated211784481737152 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 133591678 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed228312742688768 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated36251222 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 149279232 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 trainer/global_step12649 _step1010 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11432712739840 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peD4 ak 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed84045170 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current246 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated11131585670656 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 133591678 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed229125647335936 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 124375552 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130Y&M PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated228312847164928 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated229125796615168 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 133601348 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed217181244454912 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed217693058971648 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11131498233856 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated49809384 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated228312847164928 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 217636848 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 133601348 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11104255617166 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed86060360 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed222888720314958 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11104341420402 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11432588364288 $ _timestamp1.785800899952075e+09 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated217181261494272 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated217181261494272 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 _runtime5612.775472845 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 217636848 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 217646520 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed36250978 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated84045172 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated217693083875328 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated211784481737152 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 133591678 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed228312742688768 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated36251222 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 149279232 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 trainer/global_step12649 _step1010 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11432712739840 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed84045170 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current246 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated11131585670656 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 133591678 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed229125647335936 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 124375552 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated86060606 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed49809382 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed229197030592000 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed86393948 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated229197135060992 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated229197135060992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11174614749696 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated223752102591408 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11147352410576 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed218022415842304 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed11476880273408 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 218479834 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed229197030592000 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed36391653 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated218022432881664 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 218489492 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed84370677 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed50002295 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current258 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated11174702179328 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed84370677 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated84370679 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11174614749696 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated11174702179328 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed212604733141472 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 % _timestamp1.7858009215213375e+09 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated03,O A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed218022415842304 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed11476880273408 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 218479834 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed229197030592000 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed36391653 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated218022432881664 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 218489492 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed84370677 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed50002295 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current258 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated11174702179328 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed84370677 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated84370679 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11174614749696 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated11174702179328 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed212604733141472 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 % _timestamp1.7858009215213375e+09 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated50002297 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated230013247740416 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated218536232598528 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 134118813 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 UADeviceStatsMonitor.on_train_b:7atch_start/requested_bytes.all.freed223751999748924 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 _step1013 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated218022432881664 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed230013087968256 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated212604750180832 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed218022415842304 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated86394208 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed11147266607452 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 134118813 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current260 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated36391911 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 trainer/global_step12699 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed229197030592000 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed86393948 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated229197135060992 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated229197135060992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11174614749696 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated223752102591408 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11147352410576 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392Š2- _wandbruntime5634  _runtime5634wKe&M J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed84377320 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated230031356537856 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed230031198870016 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated223769720237062 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11148144080674 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated218039599644672 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated212621490353152 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed229215076974592 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed218039582605312 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 134129350 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _step1014 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 218497000 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated84377322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed223769617394466 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated11175581806080 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated84377322 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated50006234 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed36394468 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated11175581806080 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed50006232 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 218506672 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed218553414811648 % _timestamp1.7858009218900805e+09 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated36394715 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 trainer/global_step12699 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed218039582605312 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current249 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated229215181450752 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 134119680 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated218039599644672 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed86400700 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed230031198870016 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated223769720237062 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11148144080674 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated218039599644672 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated212621490353152 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed229215076974592 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed218039582605312 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 134129350 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _step1014 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 218497000 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated84377322 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed223769617394466 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated11175581806080 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated84377322 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated50006234 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed36394468 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated11175581806080 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed50006232 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 218506672 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed218553414811648 % _timestamp1.7858009218900805e+09 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated36394715 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 trainer/global_step12699 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed218039582605312 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current249 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated229215181450752 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 134119680 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated218039599644672 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed86400700 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 FDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated84702829 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 $ _timestamp1.785800943900258e+09 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed230099364877824 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated218880771032064 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.curLnrent17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current258 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated11218698314752 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11522172419584 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated213441758796832 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 134637159 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 141159936 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11218610885120 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed36534935 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated219396588438528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130U'O L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed84702827 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed219396563534848 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated224632999670916 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed84702827 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed230099364877824 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 219349644 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated84702829 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed86734080 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed50199145 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 166063616 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 trainer/global_step12749 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated218880771032064 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed224632896828432 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated219396588438528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11218610885120 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed11191155070960 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 _runtime5656.723524374 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 134646815 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated84702829 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 $ _timestamp1.785800943900258e+09 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed230099364877824 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated218880771032064 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current258 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated11218698314752 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11522172419584 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated213441758796832 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 134637159 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 141159936 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11218610885120 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed36534935 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 219366824 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed213458481929792 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated36538089 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 _runtime5657.099259006 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed84709470 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed224650514473974 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed218897920755712 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current270 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11192032544182 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated84709472 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated218897937795072 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed36537821 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 134647682 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated213458498969152 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11192118347418 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated84709472 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated219413795555328 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 trainer/global_step12749 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated230936863151104 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 219366824 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed50203082 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated11219577941504 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed230936707580416 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 134657352 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed218897920755712 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed84709470 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current268 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11522936928768 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed86740903 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated230117515736576 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated230936863151104 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 219366824 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed50203082 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated11219577941504 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed230936707580416 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 134657352 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed218897920755712 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed84709470 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current268 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11522936928768 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed86740903 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated230117515736576 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 219366824 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed213458481929792 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated36538089 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 _runtime5657.099259006 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed84709470 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed224650514473974 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed218897920755712 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current270 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11192032544182 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated84709472 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated218897937795072 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed36537821 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 134647682 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated213458498969152 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11192118347418 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated84709472 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated219413795555328 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 trainer/global_step12749 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 F  /loss/FRt2/detection_loss2.2996710567589265 .loss/FBt1/detection_loss0.568440678829868 /loss/FRt1/assignment_loss2.624571582988689 0loss/FRt2/assignment_loss2.1870807919157556 3loss/FBt1/assignment_loss0.0011667777913725416 &loss/total_loss15.347857220106944 epoch0 % _timestamp1.7858009442777495e+09 _runtime 5657.10063351 _step1019 1loss/SRqqt1/detection_loss1.4254590341464637 0loss/SRqqt2/detection_loss1.085239474590555 2loss/FBt2/assignment_loss0.001450673889858824 /loss/FRt1/detection_loss2.5932355174108555 /loss/FBt2/detection_loss1.1793186087172496 trainer/global_step12749 2loss/SRqqt1/assignment_loss0.6114645473500515 2loss/SRqqt2/assignment_loss0.7707584757172993SK % _timestamp1.7858009442777495e+09 _runtime 5657.10063351 _step1019 1loss/SRqqt1/detection_loss1.4254590341464637 0loss/SRqqt2/detection_loss1.085239474590555 2loss/FBt2/assignment_loss0.001450673889858824 /loss/FRt1/detection_loss2.5932355174108555 /loss/FBt2/detection_loss1.1793186087172496 trainer/global_step12749 2loss/SRqqt1/assignment_loss0.6114645473500515 2loss/SRqqt2/assignment_loss0.7707584757172993 /loss/FRt2/detection_loss2.2996710567589265 .loss/FBt1/detection_loss0.568440678829868 /loss/FRt1/assignment_loss2.624571582988689 0loss/FRt2/assignment_loss2.1870807919157556 3loss/FBt1/assignment_loss0.0011667777913725416 &loss/total_loss15.347857220106944 epoch07BU(:  ȰW disk.dm-0.in 4.64453125$ disk.nvme0n1.out263.39794921875 disk.drbd2111.out0 network.sent 1.1717138e+07 disk.sda1.out0 network.recv669724$ disk./.usageGB578.3500480651855 disk.drbd1938.out0 disk.sda2.out 153.1328125 disk.drbd1965.out0 disk.drbd1828.in0 disk.drbd2078.out0 disk.drbd1552.out0 disk.drbd1285.in0 disk.drbd2078.in0 disk.drbd1584.out0 disk.drbd2111.in0* disk./.usagePercent32.344362322270804 disk.drbd1965.in0 disk.sda2.in0! proc.memory.rssMB 11260.40625 cpu13.7541241962696 disk.drbd2041.in0 disk.drbd1828.out0 disk.rbd0.in0 disk.sda.in0 disk.nvme0n1.in 4.64453125 disk.drbd1817.out0* proc.memory.percent17.182016372680664 disk.drbd1817.in0 disk.drbd1285.out0! disk.dm-0.out263.39794921875 disk.drbd1226.in0 disk.drbd1226.out0 disk.drbd1584.in0 disk.drbd1552.in0 disk.drbd1289.in0 disk.drbd1289.out0 disk.drbd1510.out0 disk.drbd2118.out0 disk.rbd0.out0 disk.drbd1356.in0 disk.rbd1.in0 disk.drbd1550.in0 disk.drbd1510.in0$ memory_percent41.24928116798401* proc.memory.availableMB38502.87109375 disk.drbd1673.in0 disk.drbd2118.in0 disk.drbd1931.in0 disk.drbd1550.out0 disk.rbd1.out22.65625 disk.drbd2041.out0 disk.drbd1938.in0 disk.drbd1356.out0 proc.cpu.threads29 disk.drbd1673.out0 disk.drbd1931.out0 disk.sda.out 153.1328125 disk.sda1.in0` (: W gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts124.938' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent41.646 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0[ky2 - _wandbruntime5678  _runtime5678t#  & lr-AdamW/pg20.0003298582148254327 trainer/global_step12799 % _timestamp1.7858009659682329e+09 _runtime5678.791269372 _step1020 & lr-AdamW/pg10.0003298582148254327Nl  % _timestamp1.7858009659682329e+09 _runtime5678.791269372 _step1020 & lr-AdamW/pg10.0003298582148254327 & lr-AdamW/pg20.0003298582148254327 trainer/global_step12799DŽ2 - _wandbruntime5678  _runtime5678I8' O JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated11262694450176 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated85034979 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 % _timestamp1.7858009659692786e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated85034979 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated231824361847296 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11235129337592 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 _runtime5678.792569901 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed85034977 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed231001699163648 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed214278750373472 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed87075109 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated219739109182464 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed231001699163648 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current276 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed231824204172288 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated11262694450176 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed36679114 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 135165161 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed50395995 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated50395997 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed11235043534468 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 _step1021 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed219739092143104 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated231001803632640 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed220256919374848 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 220200138 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 trainer/global_step12799 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current278 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated36679390 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated214278767412832 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 135165161 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed50395995 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated50395997 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed11235043534468 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 _step1021 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed219739092143104 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated231001803632640 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed220256919374848 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 220200138 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 trainer/global_step12799 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current278 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated36679390 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated214278767412832 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 % _timestamp1.7858009659692786e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated85034979 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated231824361847296 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11235129337592 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 _runtime5678.792569901 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed85034977 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed231001699163648 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed214278750373472 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed87075109 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated219739109182464 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121c92 - _wandbruntime5679  _runtime5679T  M [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated219756275945472 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 220217304 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated85041622 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated231019850022400 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 % _timestamp1.7858009663434427e+09 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current235 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 135185354 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11235921007690 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated87082069 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 _runtime5679.166868503 _step1022 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated225531514396078 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed219756258906112 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed50399932 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KEDeviceStaztsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 135185354 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated36682135 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11568184764928 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 220226976 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed36681900 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated11263574076928 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed225531411553482 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed231019745546240 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 135175684 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed87081832 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed231842311256576 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed220274126491648 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed214295490545792 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed85041620 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current237 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 220226976 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated231842471021568 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated231019850022400 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11568319626240 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 trainer/global_step12799 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 135175684 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11236006810926 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032X[& M E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated87082069 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 _runtime5679.166868503 _step1022 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated225531514396078 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed219756258906112 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed50399932 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 135185354 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated36682135 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11568184764928 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 220226976 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed36681900 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated11263574076928 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed225531411553482 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed231019745546240 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 135175684 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed87081832 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed231842311256576 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed220274126491648 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed214295490545792 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed85041620 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current237 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 220226976 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated231842471021568 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated231019850022400 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11568319626240 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 trainer/global_step12799 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 135175684 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11236006810926 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated219756275945472 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 220217304 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated85041622 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated231019850022400 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 % _timestamp1.7858009663434427e+09 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current235 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 135185354 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11235921007690 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 u2 - _wandbruntime5679  _runtime5679kL   _step1023 3loss/FBt1/assignment_loss2.166862926096987e-05 .loss/FRt2/detection_loss2.176705314057697 1loss/SRqqt1/detection_loss1.2615056763046442 epoch0 0loss/SRqqt1/assignment_loss0.84804370598818 3loss/FBt2/assignment_loss0.0036198508323497824 /loss/FRt1/detection_loss2.4628258947791024 1loss/SRqqt2/detection_loss0.9118879672102127 /loss/FBt1/detection_loss0.5934223185759038 trainer/global_step12799 % _timestamp1.7858009663451848e+09 _runtime5679.168028652 0loss/FRt2/assignment_loss2.2055880875931573 0loss/SRqqt2/assignment_loss0.79532108788152 &loss/total_loss14.905467011090735 0loss/FRt1/assignment_loss2.7263214093717663 /loss/FBt2/detection_loss0.9202040298669426  0loss/FRt1/assignment_loss2.7263214093717663 /loss/FBt2/detection_loss0.9202040298669426 _step1023 3loss/FBt1/assignment_loss2.166862926096987e-05 .loss/FRt2/detection_loss2.176705314057697 1loss/SRqqt1/detection_loss1.2615056763046442 epoch0 0loss/SRqqt1/assignment_loss0.84804370598818 3loss/FBt2/assignment_loss0.0036198508323497824 /loss/FRt1/detection_loss2.4628258947791024 1loss/SRqqt2/detection_loss0.9118879672102127 /loss/FBt1/detection_loss0.5934223185759038 trainer/global_step12799 % _timestamp1.7858009663451848e+09 _runtime5679.168028652 0loss/FRt2/assignment_loss2.2055880875931573 0loss/SRqqt2/assignment_loss0.79532108788152 &loss/total_loss14.905467011090735i (: U gpu.0.gpu23.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts111.711' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent37.236999999999995 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0;Z (:  U disk.dm-0.in 4.64453125 disk.drbd2118.in0 disk.drbd2118.out0 proc.cpu.threads29 network.recv671411 disk.drbd1931.out0 disk.drbd1828.out0 disk.drbd1550.in0 disk.drbd1938.in0 disk.rbd0.out0 disk.drbd1552.out0 disk.drbd1289.out0 disk.drbd2078.out0 disk.drbd1550.out0 disk.drbd1938.out0 disk.sda.in0 disk.drbd1510.in0 disk.drbd2078.in0 disk.drbd1965.in0 disk.sda1.out0 disk.drbd1289.in0 disk.drbd1584.out0 disk.drbd1356.out0 disk.sda2.in0 disk.drbd2111.in0 disk.drbd2041.in0 disk.sda.out 153.5078125$ disk.nvme0n1.out264.11767578125 disk.drbd1931.in0 disk.drbd1510.out0! disk.dm-0.out264.11767578125$ disk./.usageGB578.3500671386719 disk.rbd0.in0 disk.drbd1226.in0 disk.sda1.in0 disk.drbd1673.in0 disk.sda2.out 153.5078125 disk.drbd1226.out0 disk.nvme0n1.in 4.64453125 disk.rbd1.out 22.6884765625) proc.memory.availableMB 38501.4140625 disk.drbd1356.in0 disk.drbd1817.out0 disk.rbd1.in0 disk.drbd1285.out0 network.sent 1.1762193e+07* disk./.usagePercent32.344363388960005 disk.drbd1673.out0 disk.drbd1552.in0 disk.drbd1828.in0$ memory_percent41.25150442123413 disk.drbd1285.in0 disk.drbd2041.out0 disk.drbd2111.out0 disk.drbd1817.in0! proc.memory.rssMB 11260.40625* proc.memory.percent17.182016372680664 cpu13.75362131162188 disk.drbd1584.in0 disk.drbd1965.out0tI (: ﾱU gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts99.983' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent33.327666666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0K45[ (:  U disk.drbd1226.out0 disk.drbd1584.out0$ disk.nvme0n1.out264.58740234375% memory_percent41.251468658447266 disk.drbd1965.in0 disk.drbd1828.in0 disk.drbd2118.in0! proc.memory.rssMB 11260.40625 network.recv672816 disk.drbd1552.out0 disk.sda.out 154.03515625 disk.drbd1510.out0 disk.drbd1226.in0 disk.sda.in0$ disk./.usageGB578.3500480651855 disk.drbd2111.out0 disk.drbd1965.out0 disk.drbd1938.in0* proc.memory.percent17.182016372680664 network.sent 1.1807352e+07 disk.drbd1673.in0 disk.drbd1289.in0 disk.drbd1285.in0 disk.nvme0n1.in 4.64453125 disk.drbd2041.in0 disk.drbd2041.out0 disk.rbd1.out 22.8681640625 disk.drbd1550.in0 disk.drbd1550.out0 disk.rbd0.in0 disk.drbd1931.in0 disk.drbd1552.in0 disk.drbd1510.in0 disk.drbd2078.out0 disk.rbd1.in0 disk.drbd1289.out0 disk.sda1.out0 disk.drbd2118.out0 disk.sda2.in0 cpu13.753264146033938 proc.cpu.threads29 disk.drbd1817.in0! disk.dm-0.out264.58740234375 disk.sda2.out 154.03515625 disk.drbd1931.out0 disk.dm-0.in 4.64453125 disk.drbd2111.in0 disk.drbd1817.out0& proc.memory.availableMB 38501.4375* disk./.usagePercent32.344362322270804 disk.drbd1673.out0 disk.drbd1938.out0 disk.drbd2078.in0 disk.drbd1828.out0 disk.drbd1584.in0 disk.drbd1356.in0 disk.drbd1356.out0 disk.rbd0.out0 disk.sda1.in0 disk.drbd1285.out0P2 - _wandbruntime5701  _runtime5701K  & lr-AdamW/pg10.0003311468241496981 & lr-AdamW/pg20.0003311468241496981 trainer/global_step12849 % _timestamp1.7858009883624737e+09 _runtime5701.185716827 _step1024   _step1024 & lr-AdamW/pg10.0003311468241496981 & lr-AdamW/pg20.0003311468241496981 trainer/global_step12849 % _timestamp1.7858009883624737e+09 _runtime5701.185716827S2 - _wandbruntime5701  _runtime57018 # O dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11612623504384 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed85367127 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated87416472 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed36823369 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed85367127 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 % _timestamp1.7858009883637543e+09 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed220597430293504 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed215115758989472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed231904033449472 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed11612492830208 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated220597447332864 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated36823625 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed232729768045056 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated220597447332864 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed220597430293504 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated221117300118528 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current256 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 221069948 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated215115776028832 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 _runtime5701.189119814 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated231904137918464 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 221060290 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 13569316DjM3 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current258 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11306603155968 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed11278931997976 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11306603155968 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856s+' O LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed231904033449472 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed11612492830208 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated220597447332864 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated36823625 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed232729768045056 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated220597447332864 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed220597430293504 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated221117300118528 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current256 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 221069948 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated215115776028832 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 _runtime5701.189119814 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated231904137918464 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 221060290 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 135693163 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current258 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11306603155968 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed11278931997976 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11306603155968 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11612623504384 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed85367127 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated87416472 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed36823369 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed85367127 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 % _timestamp1.7858009883637543e+09 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed220597430293504 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed215115758989472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 Jn& M J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated231922184308224 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11307482775552 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 % _timestamp1.7858009887371125e+09 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 135713356 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 135713356 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current245 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated36826431 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated232748034304512 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 221087128 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated85373772 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed50596782 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed85373770 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11613396402176 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated220614614095872 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed221134482331648 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed226412308632990 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 _runtime5701.560512479 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11613527069184 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11279809471198 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated11307570212352 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 trainer/global_step12849 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 221077456 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated226412411475586 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 221087128 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated220614614095872 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated231922184308224 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed215132499161792 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 135703686 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed87422968 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 _step1026 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 135703686 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated87423215 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed231922079832064 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed85373770 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated221134507235328 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed11307482775552 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed232747878733824 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed220614597056512 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed36826186 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032Wj& M ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated11307570212352 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated85373772 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated215132516201152 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 221087128 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated85373772 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed50596782 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed85373770 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11613396402176 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated220614614095872 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed221134482331648 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed226412308632990 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 _runtime5701.560512479 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11613527069184 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11279809471198 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated11307570212352 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 trainer/global_step12849 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 221077456 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated226412411475586 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 221087128 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated220614614095872 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated231922184308224 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed215132499161792 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 135703686 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed87422968 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 _step1026 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 135703686 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated87423215 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed231922079832064 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed85373770 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated221134507235328 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed11307482775552 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed232747878733824 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed220614597056512 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed36826186 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated231922184308224 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11307482775552 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 % _timestamp1.7858009887371125e+09 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 135713356 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 135713356 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current245 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated36826431 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated232748034304512 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 EDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 _step1029 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed85699277 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed50789695 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated227275690909440 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed87756162 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11350599291392 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated11350686721024 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed11657678517248 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 _runtime5723.772669332 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated85699279 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed11322820461484 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated85699279 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated232806472204288 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated233635473538560 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11350599291392 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed232806367735296 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated221455785483264 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11657817580032 $ _timestamp1.785801010945958e+09 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 136230821 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated221455785483264 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 221930100 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed232806367735296 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated221455785483264 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11657817580032 $ _timestamp1.785801010945958e+09 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 136230821 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated221455785483264 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 221930100 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 _step1029 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed85699277 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed50789695 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated227275690909440 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed87756162 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11350599291392 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated11350686721024 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed11657678517248 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 _runtime5723.772669332 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated85699279 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed11322820461484 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated85699279 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated232806472204288 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated23363547353856082 - _wandbruntime5724  _runtime5724% M RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _step1030 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 136231688 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated11351566347776 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11351478910976 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 136231688 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed233653422505472 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11658717097984 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 % _timestamp1.7858010113175266e+09 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11658584333824 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated215969524817152 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated232824518594048 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 221937608 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated221472952246272 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated233653580173312 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed221994838171648 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 _runtime5724.140904228 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed87762948 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated221472952246272 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed227293205712498 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed11351478910976 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed85705920 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated11351566347776 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed215969507777792 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed221472935206912 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed36969316 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed85705920 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed221472935206912 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 136241358 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated50793634 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current270 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 136241358 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated85705922 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed50793632 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 221947280 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated87763220 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current272 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 J;DeviceStatsMonitor.on_train_batch_end/activeP{_bytes.all.peak 2940455936 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed232824414117888 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated227293308555094 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated221994863075328 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032#SZ& M >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed221472935206912 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed36969316 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed85705920 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed221472935206912 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 136241358 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated50793634 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current270 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 136241358 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated85705922 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed50793632 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 221947280 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated87763220 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current272 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed232824414117888 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated227293308555094 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated221994863075328 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _step1030 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 136231688 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated11351566347776 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11351478910976 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 136231688 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed233653422505472 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11658717097984 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 % _timestamp1.7858010113175266e+09 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11658584333824 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated215969524817152 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated232824518594048 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 221937608 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated221472952246272 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated233653580173312 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed221994838171648 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 _runtime5724.140904228 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed87762948 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated221472952246272 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed227293205712498 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed11351478910976 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed85705920 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated11351566347776 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed2159695077777922 - _wandbruntime5724  _runtime5724&/t   trainer/global_step12899 /loss/FRt1/detection_loss2.3338414480288825 1loss/SRqqt1/detection_loss1.2532351226028469 1loss/SRqqt2/detection_loss1.0112672073185338 /loss/FBt1/detection_loss0.8329160427995796 % _timestamp1.7858010113192248e+09 3loss/FBt1/assignment_loss0.0018411599760920561 3loss/FBt2/assignment_loss7.012310732079641e-09 .loss/FRt2/detection_loss2.156841493171194 %loss/total_loss14.61939192251561 _runtime5724.142194511 _step1031 /loss/FRt2/assignment_loss2.189690736324891 epoch0 .loss/FRt1/assignment_loss2.21913173298041 2loss/SRqqt1/assignment_loss0.9003177862170906 2loss/SRqqt2/assignment_loss1.0152365768143365 /loss/FBt2/detection_loss0.7050726092694437@f  epoch0 .loss/FRt1/assignment_loss2.21913173298041 2loss/SRqqt1/assignment_loss0.9003177862170906 2loss/SRqqt2/assignment_loss1.0152365768143365 /loss/FBt2/detection_loss0.7050726092694437 trainer/global_step12899 /loss/FRt1/detection_loss2.3338414480288825 1loss/SRqqt1/detection_loss1.2532351226028469 1loss/SRqqt2/detection_loss1.0112672073185338 /loss/FBt1/detection_loss0.8329160427995796 % _timestamp1.7858010113192248e+09 3loss/FBt1/assignment_loss0.0018411599760920561 3loss/FBt2/assignment_loss7.012310732079641e-09 .loss/FRt2/detection_loss2.156841493171194 %loss/total_loss14.61939192251561 _runtime5724.142194511 _step1031 /loss/FRt2/assignment_loss2.189690736324891vx (: ҾU gpu.0.gpu38.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts99.359' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent33.11966666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0WL (:  ۀV disk.drbd1673.out0 disk.dm-0.out 265.5390625! proc.memory.rssMB 11260.40625 disk.drbd2118.out0 disk.drbd1552.in0 disk.rbd0.out0 disk.sda1.out0 disk.drbd1510.in0 disk.drbd2078.out0 disk.drbd1931.out0 disk.drbd1584.out0 disk.sda2.out 155.21875 disk.drbd1828.out0 disk.drbd1550.out0 network.sent 1.18556e+07 disk.drbd1356.in0 disk.drbd1817.in0 disk.drbd1938.out0 disk.sda1.in0 disk.drbd1226.in0 disk.drbd1226.out0) proc.memory.availableMB 38501.6328125 cpu13.753268509491626 disk.sda2.in0 disk.drbd1931.in0 disk.drbd1289.in0 disk.drbd1289.out0 disk.drbd1673.in0 disk.drbd2041.in0 disk.nvme0n1.in 4.64453125 disk.dm-0.in 4.64453125$ memory_percent41.25117063522339) disk./.usagePercent32.34436360229785$ disk./.usageGB578.3500709533691 disk.drbd2118.in0 disk.rbd1.in0 disk.sda.out 155.21875 network.recv675839 disk.rbd1.out 22.9306640625 disk.drbd1552.out0 disk.drbd1965.out0 disk.drbd1828.in0 disk.drbd1285.in0 disk.drbd2111.in0 disk.rbd0.in0* proc.memory.percent17.182016372680664 disk.drbd1965.in0 disk.drbd1550.in0 disk.drbd1285.out0 proc.cpu.threads29 disk.drbd1584.in0 disk.drbd1356.out0 disk.drbd2111.out0 disk.drbd1817.out0 disk.sda.in0 disk.drbd1510.out0 disk.drbd2078.in0 disk.nvme0n1.out 265.5390625 disk.drbd2041.out0 disk.drbd1938.in0Z. (: պU gpu.0.gpu35.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts109.921' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent36.64033333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0)V (:  ڃV disk.drbd1584.in0 disk.drbd1550.out0* proc.memory.percent17.182016372680664 network.recv677470 disk.drbd1356.in0 disk.dm-0.in 4.64453125 network.sent 1.1900596e+07 disk.nvme0n1.in 4.64453125# disk.nvme0n1.out265.7607421875 disk.drbd1965.in0 disk.drbd1550.in0 proc.cpu.threads29 disk.drbd2118.out0 disk.drbd1356.out0 disk.drbd2111.out0 disk.rbd1.out 22.962890625 cpu13.752743873266217 disk.drbd1828.in0 disk.rbd0.in0 disk.sda.out 155.46484375 disk.drbd1285.out0% proc.memory.availableMB 38501.375 disk.drbd1817.in0 disk.drbd1965.out0 disk.drbd1289.in0 disk.sda2.out 155.46484375 disk.drbd1817.out0 disk.drbd1931.in0 disk.drbd1931.out0 disk.rbd1.in0 disk.drbd1673.out0 disk.drbd2041.out0 disk.drbd1552.in0 disk.drbd1285.in0$ disk./.usageGB578.3500709533691 disk.drbd1289.out0 disk.drbd1938.out0 disk.sda1.in0 disk.dm-0.out265.7607421875% memory_percent41.251564025878906 disk.drbd1584.out0 disk.drbd1510.in0! proc.memory.rssMB 11260.40625 disk.drbd2078.in0 disk.sda2.in0 disk.drbd2111.in0 disk.drbd2041.in0 disk.drbd1673.in0 disk.drbd1552.out0 disk.drbd1938.in0 disk.sda.in0 disk.sda1.out0 disk.drbd1510.out0 disk.drbd1226.in0) disk./.usagePercent32.34436360229785 disk.drbd1828.out0 disk.rbd0.out0 disk.drbd1226.out0 disk.drbd2118.in0 disk.drbd2078.out0)2 - _wandbruntime5746  _runtime5746  % _timestamp1.7858010336383426e+09 _runtime5746.461314285 _step1032 & lr-AdamW/pg10.0003337240427982287 & lr-AdamW/pg20.0003337240427982287 trainer/global_step129493d  _runtime5746.461314285 _step1032 & lr-AdamW/pg10.0003337240427982287 & lr-AdamW/pg20.0003337240427982287 trainer/global_step12949 % _timestamp1.7858010336383426e+09^2 - _wandbruntime5746  _runtime5746yj' O VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated233708806490112 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed233708702021120 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed233708702021120 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated11394682856448 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed50986545 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated234540994331648 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 136758823 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated228156587988948 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated11394682856448 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated37111146 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11702982533120 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed11702853956096 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed228156485146464 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated222314123633664 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed86031427 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 _step1033 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated222838011798528 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed216789776221472 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11394595426816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed11366708924992 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current260 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated233708806490112 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed86031427 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 136749167 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed222837986894848 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 $ _timestamp1.785801033639317e+09 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 222780594 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated88097693 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed88097433 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11366794728116 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated50986547 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed37110888 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated86031429 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current258 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 n O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 _runtime5746.462601898 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed234540840850944 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed222314106594304 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 222790252 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated86031429 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11394595426816 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 $ _timestamp1.785801033639317e+09 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 222780594 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated88097693 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed88097433 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11366794728116 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated50986547 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed37110888 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated86031429 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current258 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated233708806490112 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed233708702021120 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed233708702021120 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated11394682856448 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed50986545 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated234540994331648 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 136758823 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 27G 68240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated228156587988948 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated11394682856448 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated37111146 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11702982533120 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed11702853956096 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed228156485146464 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated222314123633664 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed86031427 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 _step1033 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated222838011798528 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed216789776221472 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11394595426816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed11366708924992 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current260 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated233708806490112 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed86031427 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 136749167 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed222837986894848 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 222807432 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated233726852879872 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated222331290396672 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 222797760 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11395475046400 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed228174102792006 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed234558955422720 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 222797760 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated11395562483200 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed88104250 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 147182080 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed222855194011648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11367672201450 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated50990484 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed86038070 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 222807432 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated216806533433152 _runtime5746.827569561 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed37113768 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated228174205634602 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 % _timestamp1.7858010340042026e+09 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 136759690 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 122278400 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11367586398214 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated222331290396672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated86038072 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11703761411072 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed50990482 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated222855218915328 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _step1034 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated11395562483200 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated222855218915328 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _step1034 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated11395562483200 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 222807432 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated233726852879872 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated222331290396672 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 222797760 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11395475046400 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed228174102792006 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed234558955422720 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 222797760 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated11395562483200 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed88104250 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 147182080 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed222855194011648 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11367672201450 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated50990484 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed86038070 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 222807432 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated216806533433152 _runtime5746.827569561 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed37113768 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated228174205634602 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 % _timestamp1.7858010340042026e+09 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 136759690 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 122278400 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11367586398214 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated222331290396672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated86038072 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11703761411072 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0'2 - _wandbruntime5746  _runtime5746J   2loss/FBt2/assignment_loss0.003909589399531053 .loss/FRt2/detection_loss2.368831617659644 1loss/SRqqt1/detection_loss1.3451054081087932 1loss/SRqqt2/detection_loss1.1786441059666686 0loss/FRt2/assignment_loss1.9254104313097502 % _timestamp1.7858010340058649e+09 _runtime5746.828687135 _step1035 2loss/SRqqt1/assignment_loss0.9137978997217336 4loss/FBt1/assignment_loss1.8612165707452277e-05 epoch0 trainer/global_step12949 0loss/FRt1/assignment_loss2.7281661428600907 2loss/SRqqt2/assignment_loss0.8191048671994524 .loss/FRt1/detection_loss2.604327968265637 .loss/FBt1/detection_loss1.004069751277566 /loss/FBt2/detection_loss1.2473623456003575 &loss/total_loss16.138748739534932R؜  1loss/SRqqt2/detection_loss1.1786441059666686 0loss/FRt2/assignment_loss1.9254104313097502 % _timestamp1.7858010340058649e+09 _runtime5746.828687135 _step1035 2loss/SRqqt1/assignment_loss0.9137978997217336 4loss/FBt1/assignment_loss1.8612165707452277e-05 epoch0 trainer/global_step12949 0loss/FRt1/assignment_loss2.7281661428600907 2loss/SRqqt2/assignment_loss0.8191048671994524 .loss/FRt1/detection_loss2.604327968265637 .loss/FBt1/detection_loss1.004069751277566 /loss/FBt2/detection_loss1.2473623456003575 &loss/total_loss16.138748739534932 2loss/FBt2/assignment_loss0.003909589399531053 .loss/FRt2/detection_loss2.368831617659644 1loss/SRqqt1/detection_loss1.3451054081087932+ w (: ޢU gpu.0.gpu24.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts127.074' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent42.358000000000004 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0X (:  U disk.drbd1673.in0 disk.rbd0.in0 disk.drbd2118.in0 disk.sda2.out 155.78515625 disk.drbd1673.out0 disk.sda.in0 disk.drbd1226.in0 network.recv678301 disk.drbd1931.out0 disk.drbd1938.in0 disk.drbd1285.in0 disk.drbd1510.in0 disk.drbd2078.out0 disk.drbd1356.in0 disk.sda2.in0 disk.rbd1.in0 disk.drbd1550.in0 disk.drbd1285.out0 disk.drbd2078.in0 disk.drbd2111.out0 proc.cpu.threads29 disk.drbd1817.in0 disk.drbd1817.out0 disk.drbd1226.out0 disk.drbd1584.in0 disk.nvme0n1.in 4.64453125* disk./.usagePercent32.344362535608646 disk.drbd2041.in0 disk.dm-0.out265.9873046875 disk.drbd1584.out0# disk.nvme0n1.out265.9873046875 disk.drbd1356.out0 disk.sda.out 155.78515625 disk.sda1.in0' proc.memory.availableMB 38501.34375$ memory_percent41.25161170959473 network.sent 1.1903612e+07 disk.drbd1965.out0 disk.drbd1828.in0 disk.drbd1289.in0 disk.drbd1938.out0! proc.memory.rssMB 11260.40625 disk.rbd1.out 23.056640625 disk.drbd1965.in0 disk.drbd1828.out0 disk.drbd2118.out0 disk.drbd1289.out0 disk.rbd0.out0 disk.drbd1510.out0 disk.dm-0.in 4.64453125* proc.memory.percent17.182016372680664$ disk./.usageGB578.3500518798828 disk.drbd1931.in0 disk.drbd2041.out0 disk.drbd1552.in0 disk.drbd1550.out0 disk.sda1.out0 disk.drbd1552.out0 disk.drbd2111.in0 cpu13.752158424103223$j2 - _wandbruntime5768  _runtime5768T  % _timestamp1.7858010555788856e+09 _runtime5768.401865877 _step1036 ' lr-AdamW/pg10.00033501265212249395 ' lr-AdamW/pg20.00033501265212249395 trainer/global_step12999u  ' lr-AdamW/pg20.00033501265212249395 trainer/global_step12999 % _timestamp1.7858010555788856e+09 _runtime5768.401865877 _step1036 ' lr-AdamW/pg10.00033501265212249395Wti2 - _wandbruntime5768  _runtime5768G O QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 trainer/global_step12999 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed88438032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed51183395 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 223650404 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak8627J58144 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated11438678991872 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 _runtime5768.403092681 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated234611140775936 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed86363577 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11410683191624 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 137286825 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed217626784837472 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed86363577 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 137277169 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current241 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed223698342734848 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated11438678991872 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed234611036306944 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated86363579 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed229037382225972 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 137286825 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed234611036306944 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated223698367638528 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 _step1037 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 % _timestamp1.7858010555798578e+09 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed37254637 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed223172444744704 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11438591562240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated88438273 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 223640746 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated223172461784064 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated51183397 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current239 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11438591562240 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed235446436684800 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed223172444744704 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11748224624128>' O N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed51183395 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 223650404 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated11438678991872 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 _runtime5768.403092681 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated234611140775936 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed86363577 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11410683191624 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 137286825 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed217626784837472 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed86363577 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 137277169 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current241 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed223698342734848 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated11438678991872 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed234611036306944 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated86363579 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed229037382225972 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 137286825 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed234611036306944 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated223698367638528 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 _step1037 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 % _timestamp1.7858010555798578e+09 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed37254637 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed223172444744704 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11438591562240 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated88438273 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 223640746 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated223172461784064 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated51183397 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current239 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11438591562240 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed235446436684800 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed223172444744704 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11748224624128 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 trainer/global_step12999 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed88438032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887##2 - _wandbruntime5768  _runtime5768|?& M K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 137287692 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 223657912 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed223189611507712 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11411474861722 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated234629187165696 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated229055102714110 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _runtime5768.774631973 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed235464546873856 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11748997022208 trainer/global_step12999 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current267 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 $ _timestamp1.785801055951245e+09 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated223715574755328 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 137287692 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated11439558618624 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed234629082689536 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11439471181824 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 223667584 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 137297362 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed223715549851648 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated217643542049152 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed217643525009792 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed88444788 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated37257721 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed229054999871514 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed51187332 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11749129786368 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 223657912 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current265 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated223189628547072 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed37257456 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed86370220 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 137297362 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated86370222 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed11439471181824 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated11439558618624 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392ߺD M JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 223657912 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current265 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated223189628547072 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed37257456 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed86370220 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 137297362 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated86370222 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed11439471181824 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated11439558618624 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 137287692 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 223657912 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed223189611507712 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11411474861722 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated234629187165696 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated229055102714110 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _runtime5768.774631973 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed235464546873856 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11748997022208 trainer/global_step12999 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.O all.current267 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 $ _timestamp1.785801055951245e+09 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated223715574755328 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 137287692 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated11439558618624 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed234629082689536 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11439471181824 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 223667584 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 137297362 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed223715549851648 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated217643542049152 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed217643525009792 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed88444788 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated37257721 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed229054999871514 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed51187332 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11749129786368 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0F2 - _wandbruntime5768  _runtime5768+3   2loss/SRqqt2/assignment_loss0.9801350627613556 2loss/FBt2/assignment_loss0.014533453324790772 /loss/FBt1/detection_loss0.7370939060762923 /loss/FBt2/detection_loss1.4339528857445758 %loss/total_loss16.85799129829652 epoch0 % _timestamp1.7858010559529119e+09 2loss/SRqqt1/assignment_loss0.8879485500685405 /loss/FRt2/detection_loss2.2438456695322153 1loss/SRqqt1/detection_loss1.7045448641758412 _step1039 0loss/FRt2/assignment_loss2.2182396257316674 trainer/global_step12999 _runtime5768.775747927 0loss/FRt1/assignment_loss3.0522680756163925 3loss/FBt1/assignment_loss0.0007039477862996588 /loss/FRt1/detection_loss2.6242095848468887 1loss/SRqqt2/detection_loss0.9605156726316567'  1loss/SRqqt1/detection_loss1.7045448641758412 _step1039 0loss/FRt2/assignment_loss2.2182396257316674 trainer/global_step12999 _runtime5768.775747927 0loss/FRt1/assignment_loss3.0522680756163925 3loss/FBt1/assignment_loss0.0007039477862996588 /loss/FRt1/detection_loss2.6242095848468887 1loss/SRqqt2/detection_loss0.9605156726316567 2loss/SRqqt2/assignment_loss0.9801350627613556 2loss/FBt2/assignment_loss0.014533453324790772 /loss/FBt1/detection_loss0.7370939060762923 /loss/FBt2/detection_loss1.4339528857445758 %loss/total_loss16.85799129829652 epoch0 % _timestamp1.7858010559529119e+09 2loss/SRqqt1/assignment_loss0.8879485500685405 /loss/FRt2/detection_loss2.2438456695322153Xz (: U gpu.0.gpu24.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts107.339' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent35.779666666666664 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0NZ (:  U disk.drbd1584.in0% memory_percent41.251713037490845 disk.drbd1285.out0 disk.dm-0.out266.2939453125 disk.drbd2078.out0 disk.drbd2111.out0 disk.rbd1.in0 network.sent 1.1948753e+07 disk.drbd1550.in0 disk.drbd1673.in0 disk.drbd1552.out0 disk.drbd1938.in0 disk.sda1.in0 disk.sda2.in0 disk.drbd1931.out0 disk.drbd1289.out0 disk.rbd0.out0 disk.drbd1584.out0 disk.drbd1817.in0 disk.drbd1673.out0 disk.drbd2041.in0 disk.drbd1552.in0 disk.drbd1828.out0 disk.sda1.out0 disk.drbd1356.in0 disk.drbd1938.out0 disk.drbd1510.in0 disk.drbd1510.out0 disk.drbd2111.in0 disk.sda.out 156.1171875 disk.drbd1285.in0 disk.drbd2118.in0! proc.memory.rssMB 11260.40625 disk.drbd1226.out0 disk.drbd2078.in0 disk.drbd1965.in0 disk.drbd1965.out0 proc.cpu.threads29 disk.drbd1550.out0 disk.rbd1.out 23.205078125 disk.drbd1226.in0* proc.memory.availableMB38501.27734375 cpu13.751660403736018 disk.drbd1289.in0 network.recv679842 disk.drbd2118.out0 disk.sda2.out 156.1171875 disk.drbd2041.out0 disk.drbd1828.in0 disk.rbd0.in0# disk.nvme0n1.out266.2939453125$ disk./.usageGB578.3499946594238 disk.drbd1356.out0* proc.memory.percent17.182016372680664 disk.sda.in0 disk.dm-0.in 4.64453125 disk.drbd1931.in0* disk./.usagePercent32.344359335541036 disk.nvme0n1.in 4.64453125 disk.drbd1817.out0  (: U gpu.0.gpu41.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts125.912' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent41.970666666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0H,U (:  V disk.dm-0.out266.5048828125 disk.drbd2078.out0# disk.nvme0n1.out266.5048828125 disk.drbd2111.out0 disk.drbd1817.in0 disk.drbd1550.out0* disk./.usagePercent32.344362535608646 disk.drbd2118.in0 disk.rbd1.out 23.205078125 disk.drbd1289.in0 disk.sda.in0 disk.sda.out 156.59375 network.sent 1.1993789e+07$ disk./.usageGB578.3500518798828 disk.drbd1931.out0 disk.drbd1584.out0 disk.nvme0n1.in 4.64453125 disk.drbd1552.out0! proc.memory.rssMB 11260.40625 disk.drbd2041.out0 disk.rbd0.in0 disk.drbd1584.in0 disk.drbd2111.in0 disk.drbd2041.in0 disk.drbd1828.out0 disk.drbd1938.out0 disk.drbd2078.in0 disk.drbd1817.out0 disk.drbd1356.in0 disk.rbd1.in0 disk.drbd1828.in0 disk.drbd1550.in0 disk.sda1.in0 disk.drbd1285.in0 disk.drbd1356.out0* proc.memory.percent17.182016372680664 disk.drbd1931.in0 disk.drbd1938.in0 disk.sda1.out0 cpu13.75107926974934 disk.drbd2118.out0 disk.sda2.in0 disk.drbd1552.in0 disk.drbd1510.out0 proc.cpu.threads29 disk.drbd1965.in0 disk.drbd1285.out0 disk.drbd1226.out0 disk.sda2.out 156.59375 disk.drbd1673.in0 disk.drbd1289.out0 disk.drbd1673.out0 disk.drbd1510.in0% memory_percent41.251832246780396 disk.dm-0.in 4.64453125* proc.memory.availableMB38501.19921875 network.recv681501 disk.drbd1226.in0 disk.drbd1965.out0 disk.rbd0.out0̵O2 - _wandbruntime5790  _runtime57909r  ' lr-AdamW/pg10.00033630126144675927 ' lr-AdamW/pg20.00033630126144675927 trainer/global_step13049 % _timestamp1.7858010780581672e+09 _runtime5790.881132204 _step1040Y  ' lr-AdamW/pg20.00033630126144675927 trainer/global_step13049 % _timestamp1.7858010780581672e+09 _runtime5790.881132204 _step1040 ' lr-AdamW/pg10.00033630126144675927KF2 - _wandbruntime5790  _runtime5790J' O XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11482587697664 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated224558723478528 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed229918279305480 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 _step1041 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed235513370592768 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 137814827 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current251 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated229918382147964 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed88778980 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated224030799934464 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed86695727 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current253 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 137814827 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 137805171 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 _runtime5790.882364734 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated235513475061760 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11482587697664 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11793459346432 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed11793330769408 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated218463810492832 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated11482675127296 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed224558698574848 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 trainer/global_step13049 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed235513370592768 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated86695729 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed218463793453472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 $ _timestamp1.785801078059156e+09 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 137805171 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated11482675127296 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed224030782895104 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated235513475061760 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 224510556 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0"' O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 137814827 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current251 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated229918382147964 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed88778980 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated224030799934464 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed86695727 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current253 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 137814827 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 137805171 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 _runtime5790.882364734 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated235513475061760 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11482587697664 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11793459346432 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed11793330769408 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated218463810492832 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated11482675127296 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed224558698574848 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 trainer/global_step13049 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed235513370592768 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated86695729 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed218463793453472 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 $ _timestamp1.785801078059156e+09 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 137805171 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated11482675127296 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed224030782895104 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated235513475061760 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 224510556 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11482587697664 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated224558723478528 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed229918279305480 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 _step1041 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed2355133705927682 - _wandbruntime5791  _runtime5791  M TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed218480533625792 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated86702372 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated224047966697472 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 224527736 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed88785819 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed224575905691648 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 trainer/global_step13049 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated11483554754048 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ]JDeviceStaˍ .tsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11455449128466 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated236370297373184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed235531416975360 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11794366777856 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed224047949658112 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11455363325230 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _runtime5791.253249186 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 137815694 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed229935896951022 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated235531521451520 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11794231916544 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated51384184 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 137815694 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11483467317248 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 137825364 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 224518064 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed86702370 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 % _timestamp1.7858010784297745e+09 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 224518064 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated86702372 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated224047966697472 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated218480550665152 _step1042 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated88786076 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated37401892 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current257 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated224575930595328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 224527736 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed51384182 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed86702370 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0& M M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 137825364 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 224518064 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed86702370 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 % _timestamp1.7858010784297745e+09 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 224518064 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated86702372 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated224047966697472 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated218480550665152 _step1042 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated88786076 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated37401892 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current257 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated224575930595328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 224527736 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed51384182 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed86702370 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed218480533625792 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated86702372 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated224047966697472 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 224527736 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed88785819 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed224575905691648 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 trainer/global_step13049 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated11483554754048 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11455449128466 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated236370297373184 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed235531416975360 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11794366777856 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed224047949658112 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11455363325230 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _runtime5791.253249186 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 137815694 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed229935896951022 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated235531521451520 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11794231916544 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated51384184 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 137815694 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11483467317248 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 BuN2 - _wandbruntime5813  _runtime5813"+  & lr-AdamW/pg20.0003375898707710246 trainer/global_step13099 % _timestamp1.7858011003448174e+09 _runtime5813.167810346 _step1044 & lr-AdamW/pg10.0003375898707710246o2#  _runtime5813.167810346 _step1044 & lr-AdamW/pg10.0003375898707710246 & lr-AdamW/pg20.0003375898707710246 trainer/global_step13099 % _timestamp1.7858011003448174e+09Χޑ2 - _wandbruntime5813  _runtime5813\p,' O QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 138342829 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed225419054414848 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 _step1045 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated237257728993280 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 225361050 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11526583833088 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated87027879 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated89120242 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed224889121045504 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current263 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated230799279227472 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated236415809347584 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 225370708 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated51577097 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed224889121045504 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 138333173 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed236415704878592 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed89119977 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 166063616 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated236415809347584 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed230799176384988 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 138342829 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11838649674752 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated224889138084864 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed237257562929664 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed87027877 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 141159936 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current265 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated87027879 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated89120242 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed224889121045504 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current263 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated230799279227472 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated236415809347584 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 225370708 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated51577097 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed224889121045504 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 138333173 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed236415704878592 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed89119977 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 166063616 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated236415809347584 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed230799176384988 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392iX2 - _wandbruntime5813  _runtime5813Ct& M A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed87034520 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed51581032 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed37545987 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed225436261531648 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 _runtime5813.539628809 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated87034522 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 225387888 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 138353366 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11839419492352 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 151376384 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated51581034 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 138343696 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated237275832400384 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 126472704 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 225378216 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed224906287808512 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed89127019 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed224906287808512 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11527463452672 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current243 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 225378216 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current241 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 $ _timestamp1.785801100716261e+09 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 225387888 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 138343696 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11499337591974 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated11527550889472 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed87034520 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 trainer/global_step13099 & M bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11839545965056 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed224906287808512 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11527463452672 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current243 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 225378216 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current241 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 $ _timestamp1.785801100716261e+09 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 225387888 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 138343696 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11499337591974 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated11527550889472 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed87034520 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 trainer/global_step13099 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed87034520 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed51581032 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed37545987 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed225436261531648 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 _runtime5813.539628809 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated87034522 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 225387888 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 138353366 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11839419492352 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 151376384 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated51581034 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 138343696 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated237275832400384 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 126472704 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 225378216 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed224906287808512 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed89127019 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 Bd  /loss/FRt1/assignment_loss2.572975817322731 trainer/global_step13099 % _timestamp1.7858011007179914e+09 _step1047 /loss/FRt2/assignment_loss1.975165359351946 2loss/SRqqt1/assignment_loss0.5910306556810004 3loss/FBt1/assignment_loss0.0010036170158697212 .loss/FRt1/detection_loss2.563328715227544 .loss/FRt2/detection_loss2.289977610759113 /loss/FBt1/detection_loss0.9848421866317159 /loss/FBt2/detection_loss1.3337697641780744 _runtime5813.540843523 2loss/SRqqt2/assignment_loss0.8681780753852884 3loss/FBt2/assignment_loss0.0008919830382591971 1loss/SRqqt2/detection_loss0.9276074064076396 &loss/total_loss15.760602753513599 epoch0 1loss/SRqqt1/detection_loss1.6518315625144169͜ (: U gpu.0.gpu37.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts125.932' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent41.977333333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0Y j[ (:  ӝV disk.rbd1.in0 disk.drbd1673.in0 disk.nvme0n1.in 4.64453125" disk.nvme0n1.out 266.966796875 disk.drbd1356.in0$ disk./.usageGB578.3500518798828 disk.drbd1550.in0 disk.rbd0.in0 disk.dm-0.in 4.64453125 disk.drbd1584.out0 disk.sda2.out 157.3203125 disk.rbd1.out 23.447265625 disk.drbd1931.in0 disk.drbd1552.in0 disk.drbd1285.out0 disk.drbd2078.out0 disk.sda2.in0* proc.memory.percent17.182016372680664 disk.drbd1938.in0 disk.drbd1938.out0 disk.drbd1356.out0 disk.drbd1828.out0 disk.drbd1550.out0 disk.drbd2041.out0 disk.drbd1552.out0 disk.drbd1828.in0 disk.drbd1226.out0 disk.dm-0.out 266.966796875 disk.drbd2118.out0 disk.sda1.in0 proc.cpu.threads29 disk.rbd0.out0 disk.drbd2111.out0! proc.memory.rssMB 11260.40625 cpu13.750032879514112 disk.drbd1817.in0 disk.drbd1289.in0 disk.drbd1226.in0* proc.memory.availableMB38500.51953125 disk.sda.out 157.3203125$ memory_percent41.25286936759949 disk.drbd1817.out0 disk.drbd1673.out0 disk.drbd1931.out0 disk.drbd1510.in0 disk.drbd2078.in0 disk.drbd2118.in0 disk.drbd2111.in0 disk.drbd1965.in0 disk.sda.in0 disk.drbd1285ǜ.in0 disk.drbd1510.out0 disk.drbd1289.out0 disk.sda1.out0 disk.drbd1584.in0 network.sent 1.2041948e+07* disk./.usagePercent32.344362535608646 disk.drbd1965.out0 disk.drbd2041.in0 network.recv684100=_^ (:  V disk.drbd1289.out0 disk.rbd1.in0 disk.drbd1550.in0 disk.drbd1550.out0 disk.sda.out 157.91015625 disk.drbd1817.in0 cpu13.749524535907518$ disk./.usageGB578.3500518798828 disk.drbd1356.in0 disk.drbd1226.in0 disk.drbd1584.out0 disk.drbd1673.in0* proc.memory.availableMB38500.73828125! proc.memory.rssMB 11260.40625 disk.drbd1510.out0 disk.drbd1552.in0 disk.drbd1510.in0 disk.drbd1965.in0 network.recv685710 disk.drbd1226.out0 disk.drbd2118.out0 disk.drbd1817.out0 disk.drbd2041.out0* disk./.usagePercent32.344362535608646 disk.drbd1828.out0 disk.rbd0.in0 disk.sda1.out0 disk.drbd1931.in0% memory_percent41.252535581588745 proc.cpu.threads29 disk.drbd1552.out0 disk.rbd0.out0 disk.drbd1285.in0 disk.sda2.in0 disk.drbd1938.in0 disk.drbd1285.out0 disk.drbd1584.in0 disk.drbd1356.out0 disk.drbd1938.out0 disk.drbd2078.in0 disk.nvme0n1.in 4.64453125$ disk.nvme0n1.out267.20361328125 disk.drbd2041.in0* proc.memory.percent17.182016372680664 disk.drbd1289.in0 disk.sda.in0 disk.drbd2078.out0 disk.drbd2111.out0 disk.drbd1965.out0 disk.drbd1828.in0 disk.sda2.out 157.91015625 disk.drbd2111.in0 disk.rbd1.out 23.509765625 disk.drbd1673.out0 network.sent 1.2087015e+07 disk.sda1.in0 disk.dm-0.in 4.64453125! disk.dm-0.out267.20361328125 disk.drbd2118.in0 disk.drbd1931.out0Px (: U gpu.0.gpu34.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts96.663' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent32.221 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0v2 - _wandbruntime5835  _runtime5835k  & lr-AdamW/pg10.0003388784800952899 & lr-AdamW/pg20.0003388784800952899 trainer/global_step13149 % _timestamp1.7858011225883343e+09 _runtime5835.412723177 _step1048'.$J  _step1048 & lr-AdamW/pg10.0003388784800952899 & lr-AdamW/pg20.0003388784800952899 trainer/global_step13149 % _timestamp1.7858011225883343e+09 _runtime5835.412723177X2 - _wandbruntime5835  _runtime5835n 2' O WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated89461331 trainer/global_step13149 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 138861175 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed225747459195904 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 226230860 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current241 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed37687143 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 126479872 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed226279410254848 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed11883660909056 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated231680176306980 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed51773945 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11570579968512 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed238163071163904 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed11542262779024 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed220137810685472 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 $ _timestamp1.785801122589028e+09 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed237318039164416 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated237318143633408 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated51773947 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11570579968512 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 138870831 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated220137827724832 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 138861175 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated37687384 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated238163222547456 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11542348582148 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 138870831 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current243 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed89461088 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated11570667398144 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated51773947 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11570579968512 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 138870831 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated220137827724832 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 138861175 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated37687384 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated238163222547456 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11542348582148 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 138870831 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current243 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed89461088 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated11570667398144 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed237318039164416 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated237318143633408 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 226238368 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 138871698 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11884698187264 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed225764625958912 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11571459588096 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed37690036 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated238181340462592 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 _step1050 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed231697691110038 trainer/global_step13149 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 226238368 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed237336085547008 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current253 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated231697793952634 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11543140252246 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed11571459588096 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 226248040 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated37690289 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated87366672 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated225764642998272 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated226296642275328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated220154567897152 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 138881368 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated237336190023168 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 226248040 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed51777882 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11543226055482 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated87366672 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated89468173 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current255 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 REDeviceStatsMonitor.on_train_batch_end/active_byte/2 s.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 138881368 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed89467918 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11884561228800 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392& M E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 _step1050 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed231697691110038 trainer/global_step13149 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 226238368 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed237336085547008 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current253 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated231697793952634 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11543140252246 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed11571459588096 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 226248040 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated37690289 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated87366672 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated225764642998272 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated226296642275328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated220154567897152 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 138881368 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated237336190023168 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 226248040 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed51777882 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11543226055482 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated87366672 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated89468173 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current255 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 138881368 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed89467918 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11884561228800 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated237336190023168 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 226238368 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 138871698 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11884698187264 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed225764625958912 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11571459588096 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed37690036 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated238181340462592wD2 - _wandbruntime5835  _runtime5835R   0loss/FRt2/assignment_loss1.8156214244663715 1loss/SRqqt2/detection_loss0.9945246922082187 /loss/FBt2/detection_loss1.1600780964829027 0loss/FRt1/assignment_loss2.9743679360339517 2loss/SRqqt1/assignment_loss0.7452828140582802 3loss/FBt2/assignment_loss0.0013519039341247207 .loss/FRt2/detection_loss2.398585849069059 1loss/SRqqt1/detection_loss1.3228320689048878 epoch0 4loss/FBt1/assignment_loss0.00012230117514008042 /loss/FRt1/detection_loss2.4942129130426207 /loss/FBt1/detection_loss0.9291464863717556 &loss/total_loss15.831316387019056 1loss/SRqqt2/assignment_loss0.995189901271743 trainer/global_step13149 % _timestamp1.7858011229640365e+09 _runtime5835.786995465 _step1051a  4loss/FBt1/assignment_loss0.00012230117514008042 /loss/FRt1/detection_loss2.4942129130426207 /loss/FBt1/detection_loss0.9291464863717556 &loss/total_loss15.831316387019056 1loss/SRqqt2/assignment_loss0.995189901271743 trainer/global_step13149 % _timestamp1.7858011229640365e+09 _runtime5835.786995465 _step1051 0loss/FRt2/assignment_loss1.8156214244663715 1loss/SRqqt2/detection_loss0.9945246922082187 /loss/FBt2/detection_loss1.1600780964829027 0loss/FRt1/assignment_loss2.9743679360339517 2loss/SRqqt1/assignment_loss0.7452828140582802 3loss/FBt2/assignment_loss0.0013519039341247207 .loss/FRt2/detection_loss2.398585849069059 1loss/SRqqt1/detection_loss1.3228320689048878 epoch0 (: ߺU gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts124.382' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent41.46066666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0iC^ (:  ЄV disk.drbd2041.out0 disk.drbd1828.in0 disk.drbd1673.in0 cpu13.749018821469342$ disk./.usageGB578.3500518798828 disk.drbd1552.in0 disk.drbd1510.out0 disk.dm-0.in 4.64453125 disk.rbd1.out 23.509765625 disk.drbd1289.in0! disk.dm-0.out267.44580078125$ disk.nvme0n1.out267.44580078125 disk.sda2.out 158.37890625 disk.drbd2111.in0 network.recv686576 disk.drbd2078.out0 disk.drbd2118.in0 disk.drbd2118.out0 disk.drbd1965.in0! proc.memory.rssMB 11260.40625 disk.drbd1226.out0 disk.drbd1673.out0% memory_percent41.252702474594116 proc.cpu.threads29 disk.rbd0.in0 disk.sda.out 158.37890625 disk.drbd1226.in0 disk.drbd1584.out0 disk.drbd1931.out0* proc.memory.percent17.182016372680664 disk.drbd1356.in0 disk.sda2.in0 disk.rbd1.in0 disk.drbd1828.out0 disk.drbd1550.out0 disk.drbd1285.out0* disk./.usagePercent32.344362535608646 disk.drbd2041.in0 disk.drbd1965.out0 disk.sda1.in0 disk.drbd1285.in0 disk.drbd2078.in0 disk.nvme0n1.in 4.64453125 disk.drbd2111.out0* proc.memory.availableMB38500.62890625 disk.sda.in0 disk.drbd1510.in0 disk.drbd1817.in0 disk.drbd1817.out0 disk.drbd1552.out0 disk.rbd0.out0 disk.drbd1356.out0 network.sent 1.2090029e+07 disk.drbd1289.out0 disk.drbd1938.out0 disk.drbd1584.in0 disk.drbd1931.in0 disk.drbd1550.in0 disk.drbd1938.in0 disk.sda1.out0q6^2 - _wandbruntime5857  _runtime5857od  & lr-AdamW/pg10.0003401670894195552 & lr-AdamW/pg20.0003401670894195552 trainer/global_step13199 % _timestamp1.7858011448555613e+09 _runtime5857.678583766 _step1052.C  & lr-AdamW/pg10.0003401670894195552 & lr-AdamW/pg20.0003401670894195552 trainer/global_step13199 % _timestamp1.7858011448555613e+09 _runtime5857.678583766 _step10522 - _wandbruntime5857  _runtime58570E' O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 trainer/global_step13199 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated87692179 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed226605797346304 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated239068767650816 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed227139766094848 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 _runtime5857.683721568 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated226605814385664 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed220974819301472 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11614576103936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated89802704 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current256 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated51970797 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11586237045656 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed232560970544004 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed51970795 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed238220373450240 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 227081354 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 139389177 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated11614663533568 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11614576103936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated37831907 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 $ _timestamp1.785801144856599e+09 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated238220477919232 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 _step1053 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 227091012 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11928976652288 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed11586151242532 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated87692179 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current258 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated226605814385664 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 139398833 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated232561073386488 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 139389177 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548D O QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed232560970544004 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed51970795 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed238220373450240 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 227081354 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 139389177 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated11614663533568 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11614576103936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated37831907 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 $ _timestamp1.785801144856599e+09 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated238220477919232 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 _step1053 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 227091012 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11928976652288 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed11586151242532 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated87692179 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current258 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated226605814385664 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 139398833 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 YEDeviceStatsMonitor.on_train_batch_start幌\ /requested_bytes.all.allocated232561073386488 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 139389177 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 trainer/global_step13199 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated87692179 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed226605797346304 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated239068767650816 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed227139766094848 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 _runtime5857.683721568 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated226605814385664 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed220974819301472 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11614576103936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated89802704 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current256 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated51970797 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11586237045656 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032)'2 - _wandbruntime5858  _runtime5858&!M NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated87698822 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _step1054 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed87698820 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated232578691032142 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 227108192 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed239086722663424 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed226622964109312 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated227156998115328 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed227156973211648 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11929749451776 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 139409370 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated11615543160320 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 139399700 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed226622964109312 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated89809446 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed89809189 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed220991559473792 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed51974732 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11615455723520 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 227098520 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated226622981148672 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed37834457 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11929880118784 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated87698822 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated37834712 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated238238524308992 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 227108192 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed87698820 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 % _timestamp1.7858011452289283e+09 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated226622981148672 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed11615455723520 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _step1054 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed87698820 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated232578691032142 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 227108192 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed239086722663424 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed226622964109312 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated227156998115328 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed227156973211648 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed11929749451776 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 139409370 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated11615543160320 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 139399700 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed226622964109312 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated89809446 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed89809189 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed220991559473792 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed51974732 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11615455723520 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 227098520 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated226622981148672 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed37834457 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11929880118784 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated87698822 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated37834712 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated238238524308992 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 227108192 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed87698820 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 % _timestamp1.7858011452289283e+09 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated226622981148672 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed11615455723520 EDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed52167645 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed239974116975616 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed239122707736064 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed90142939 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11658572239360 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated90143202 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed227464135496704 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _step1057 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 227941506 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed227464135496704 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated11658659668992 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 trainer/global_step13249 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 227951164 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated227464152536064 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 139926835 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated37975555 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated239974270456320 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated233441970465996 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed88024327 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed11973995040768 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated88024329 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated239122812205056 R>DevxHiceStatsMonitor.on_train_batch_start/active_bytes.all.freed239122707736064 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11974123617792 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11630125509164 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current261 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 % _timestamp1.7858011666332877e+09 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current263 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed228000121934848 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated239122812205056 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed221811827917472 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0]$''!O O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated88024329 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed88024327 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed37975294 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 _runtime5879.456731784 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11658572239360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed52167645 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed239974116975616 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed239122707736064 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed90142939 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11658572239360 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated90143202 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed227464135496704 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _step1057 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 227941506 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed227464135496704 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated11658659668992 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 trainer/global_step13249 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 227951164 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated227464152536064 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 139926835 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated37975555 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated239974270456320 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated233441970465996 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed88024327 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed11973995040768 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated88024329 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated239122812205056 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed239122707736064 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated11974123617792 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11630125509164 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current261 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 % _timestamp1.7858011666332877e+09 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current263 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed228000121934848 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated239122812205056 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed221811827917472 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032Lpf2!- _wandbruntime5879  _runtime5879&!M >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated52171584 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated227481319299072 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed90149645 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed37978063 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated88030972 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated11659539295744 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated11659539295744 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated233459588111650 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed233459485269054 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed52171582 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 227958672 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 139937372 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current246 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed239140754118656 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 _step1058 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated88030972 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated37978307 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 227968344 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 227968344 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11659451858944 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 227958672 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated227481319299072 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed228017329051648 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11975027362816 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11631002982498 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 139927702 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 _runtime 5879.82722093 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated239140858594816 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated228017353955328 trainer/global_step13249 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed221828568089792 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 139927702 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated239140858594816 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 % _timestamp1.7858011670038807e+09 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0$_J7 !M NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated88030972 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated11659539295744 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated11659539295744 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated233459588111650 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed233459485269054 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed52171582 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 227958672 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 139937372 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current246 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed239140754118656 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 _step1058 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated88030972 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated37978307 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 227968344 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 227968344 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11659451858944 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 227958672 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated227481319299072 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed228017329051648 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated11975027362816 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11631002982498 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 139927702 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 _runtime 5879.82722093 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated239140858594816 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated228017353955328 trainer/global_step13249 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed221828568089792 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 139927702 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated239140858594816 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 % _timestamp1.7858011670038807e+09 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated52171584 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated227481319299072 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed90149645 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed37978063 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0Xq2!- _wandbruntime5879  _runtime58799g ! /loss/FRt2/assignment_loss2.111165489805372 .loss/FRt1/detection_loss2.518113966157407 % _timestamp1.7858011670055313e+09 /loss/FRt1/assignment_loss2.480028181900213 .loss/FRt2/detection_loss2.573724970613655 1loss/SRqqt1/detection_loss1.2576370585452894 &loss/total_loss15.814778459981605 epoch0 trainer/global_step13249 1loss/SRqqt1/assignment_loss0.727708021259128 2loss/SRqqt2/assignment_loss1.1922406579721077 4loss/FBt2/assignment_loss1.8971692399891015e-05 0loss/SRqqt2/detection_loss1.071844920599512 /loss/FBt2/detection_loss1.1213617168955112 _step1059 3loss/FBt1/assignment_loss0.0018515033157260666 /loss/FBt1/detection_loss0.7590830012252836 _runtime5879.828357829cw! &loss/total_loss15.814778459981605 epoch0 trainer/global_step13249 1loss/SRqqt1/assignment_loss0.727708021259128 2loss/SRqqt2/assignment_loss1.1922406579721077 4loss/FBt2/assignment_loss1.8971692399891015e-05 0loss/SRqqt2/detection_loss1.071844920599512 /loss/FBt2/detection_loss1.1213617168955112 _step1059 3loss/FBt1/assignment_loss0.0018515033157260666 /loss/FBt1/detection_loss0.7590830012252836 _runtime5879.828357829 /loss/FRt2/assignment_loss2.111165489805372 .loss/FRt1/detection_loss2.518113966157407 % _timestamp1.7858011670055313e+09 /loss/FRt1/assignment_loss2.480028181900213 .loss/FRt2/detection_loss2.573724970613655 1loss/SRqqt1/detection_loss1.2576370585452894o'9!(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts131.921' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent43.97366666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0 Z!(:  V disk.sda1.in0# disk.nvme0n1.out267.8564453125 disk.drbd1817.in0 disk.drbd1673.in0 disk.drbd1285.out0 disk.drbd1226.out0 disk.nvme0n1.in 4.64453125 disk.drbd1828.in0 disk.sda.out 158.92578125 disk.drbd2118.out0$ disk./.usageGB578.3500747680664 disk.drbd1552.in0 disk.drbd1552.out0 disk.drbd1550.in0 disk.drbd1938.in0 disk.drbd1938.out0 disk.rbd1.in0 disk.drbd1965.out0 disk.drbd1289.out0 disk.drbd2111.in0 disk.drbd2111.out0 disk.drbd1931.in0 network.sent 1.2180273e+07 disk.dm-0.in 4.64453125 disk.drbd2118.in0 disk.rbd0.out0 disk.drbd2078.in0 disk.drbd1584.out0 disk.drbd1965.in0# memory_percent41.2533700466156 disk.drbd1584.in0 disk.sda2.out 158.92578125 disk.rbd1.out 23.6357421875) disk./.usagePercent32.34436381563569 disk.drbd1289.in0 disk.drbd1510.in0 disk.drbd2078.out0 disk.sda2.in0 disk.drbd1817.out0* proc.memory.availableMB38500.19140625! proc.memory.rssMB 11260.40625* proc.memory.percent17.182016372680664 disk.drbd2041.out0 disk.sda1.out0 disk.drbd1285.in0 disk.drbd1356.in0 network.recv689777 disk.drbd1510.out0 disk.dm-0.out267.8564453125 disk.drbd1226.in0 disk.drbd1356.out0 cpu13.748266867513568 disk.sda.in0 disk.drbd1673.out0 disk.drbd1931.out0 proc.cpu.threads29 disk.drbd2041.in0 disk.drbd1828.out0 disk.drbd1550.out0 disk.rbd0.in0ː!(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts125.286' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent41.762 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0ũ[!(:  ǀU network.sent 1.2225507e+07 disk.drbd2111.out0 disk.drbd1828.in0 disk.drbd1510.out0 disk.dm-0.in 4.64453125 disk.drbd2078.out0 disk.nvme0n1.in 4.64453125 disk.drbd1673.out0$ memory_percent41.25304818153381 disk.drbd2118.out0# disk.nvme0n1.out268.3330078125$ disk./.usageGB578.3500175476074 disk.drbd1226.in0 cpu13.747789886988645 disk.drbd1552.out0 disk.sda.in0* proc.memory.percent17.182016372680664 disk.drbd1965.in0 disk.rbd0.in0 disk.dm-0.out268.3330078125 proc.cpu.threads29 disk.drbd1817.in0 disk.sda1.in0 disk.drbd1965.out0 disk.drbd1356.in0 disk.drbd2118.in0 disk.drbd2111.in0 disk.drbd1550.in0 disk.drbd1938.out0 disk.rbd0.out0 disk.drbd1226.out0 disk.drbd1817.out0 disk.drbd1285.in0) disk./.usagePercent32.34436061556808 disk.drbd1584.out0 disk.drbd1931.in0* proc.memory.availableMB38500.40234375! proc.memory.rssMB 11260.40625 disk.rbd1.in0 disk.drbd1931.out0 disk.drbd1550.out0 disk.rbd1.out 23.6982421875 disk.drbd1356.out0 disk.drbd2078.in0 disk.sda2.out 159.19921875 disk.sda1.out0 disk.drbd1510.in0 disk.sda2.in0 disk.drbd1552.in0 disk.drbd1828.out0 disk.drbd1289.in0 disk.drbd1938.in0 network.recv691673 disk.drbd1584.in0 disk.drbd1673.in0 disk.drbd2041.in0 disk.drbd1289.out0 disk.sda.out 159.19921875 disk.drbd1285.out0 disk.drbd2041.out0W92!- _wandbruntime5901  _runtime59016! _runtime5901.457533579 _step1060 ' lr-AdamW/pg10.00034274430806808577 ' lr-AdamW/pg20.00034274430806808577 trainer/global_step13299 % _timestamp1.7858011886345527e+09P! ' lr-AdamW/pg10.00034274430806808577 ' lr-AdamW/pg20.00034274430806808577 trainer/global_step13299 % _timestamp1.7858011886345527e+09 _runtime5901.457533579 _step1060"-G2!- _wandbruntime5901  _runtime5901]'!O ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated228322490686464 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated228322490686464 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed234322764703020 _runtime5901.461518689 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed90483483 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed88356477 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 % _timestamp1.7858011886354878e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated240025146490880 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed228322473647104 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 140445181 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated38119224 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 140445181 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed240025042021888 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current238 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12019376143872 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated90483721 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated52364497 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11702568374784 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 140454837 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12019245469696 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated234322867545504 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated88356479 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed228322473647104 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed228860477774848 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656xT'!O XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11702568374784 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated11702655804416 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11674013972672 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed11673928169548 trainer/global_step13299 _step1061 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 140454837 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed88356477 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 140445181 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed240025042021888 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current238 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12019376143872 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated90483721 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated52364497 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11702568374784 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 140454837 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12019245469696 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated234322867545504 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated88356479 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed228322473647104 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed228860477774848 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated228322490686464 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated228322490686464 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed234322764703020 _runtime5901.461518689 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed90483483 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed88356477 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 % _timestamp1.7858011886354878e+09 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated240025146490880 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed228322473647104 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 140445181 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated38119224 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 JDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated11703535431168 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated52368434 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed222665576705792 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11703447994368 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 140455704 trainer/global_step13299 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed88363120 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed52368432 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current242 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 228818824 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated240897992944640 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed240897839471104 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed234340382348562 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 140455704 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 228828496 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed90490287 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 _step1062 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 140465374 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 H:DeviceStatsMonitor.on_train_batch_end/:?iactive.all.allocated 228828496 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed88363120 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed11703447994368 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed228877684891648 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated234340485191158 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11674891446006 _runtime5901.824609392 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed228339640410112 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated222665593745152 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated88363122 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 % _timestamp1.7858011890012305e+09 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12020283149312 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 140465374 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed240043088404480 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated90490531 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated38122097 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12020154579456 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 228828496 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed90490287 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 _step1062 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 140465374 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 228828496 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed88363120 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed11703447994368 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed228877684891648 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated234340485191158 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11674891446006 _runtime5901.824609392 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed228339640410112 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated222665593745152 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated88363122 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 % _timestamp1.7858011890012305e+09 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12020283149312 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 140465374 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed240043088404480 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated90490531 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated38122097 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12020154579456 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated11703535431168 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated52368434 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed222665576705792 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11703447994368 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 140455704 trainer/global_step13299 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed88363120 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed52368432 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current242 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 228818824 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated240897992944640 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed240897839471104 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed234340382348562 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 140455704zQ2!- _wandbruntime5901  _runtime5901J ! _step1063 /loss/FRt1/assignment_loss2.737051039536794 2loss/SRqqt1/assignment_loss0.7226659991219639 3loss/FBt2/assignment_loss2.413946668777811e-06 /loss/FRt1/detection_loss2.5670718679825466 1loss/SRqqt2/detection_loss0.9428419970168636 % _timestamp1.7858011890030167e+09 _runtime5901.825862061 2loss/SRqqt2/assignment_loss0.8639840650597009 .loss/FRt2/detection_loss2.470107889891445 1loss/SRqqt1/detection_loss1.2446965359419118 /loss/FBt1/detection_loss0.9342392533019718 &loss/total_loss14.994157304494507 trainer/global_step13299 0loss/FBt1/assignment_loss9.693812613319e-06 0loss/FRt2/assignment_loss1.9070659465604014 /loss/FBt2/detection_loss0.6044206023216248 epoch0 ! /loss/FRt1/assignment_loss2.737051039536794 2loss/SRqqt1/assignment_loss0.7226659991219639 3loss/FBt2/assignment_loss2.413946668777811e-06 /loss/FRt1/detection_loss2.5670718679825466 1loss/SRqqt2/detection_loss0.9428419970168636 % _timestamp1.7858011890030167e+09 _runtime5901.825862061 2loss/SRqqt2/assignment_loss0.8639840650597009 .loss/FRt2/detection_loss2.470107889891445 1loss/SRqqt1/detection_loss1.2446965359419118 /loss/FBt1/detection_loss0.9342392533019718 &loss/total_loss14.994157304494507 trainer/global_step13299 0loss/FBt1/assignment_loss9.693812613319e-06 0loss/FRt2/assignment_loss1.9070659465604014 /loss/FBt2/detection_loss0.6044206023216248 epoch0 _step1063Ke!(: U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts112.639' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent37.54633333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0RʯZ!(:  U) disk./.usagePercent32.34436274894649$ disk./.usageGB578.3500556945801 disk.drbd2078.in0 disk.rbd1.out 23.6982421875 disk.drbd1931.in0 disk.drbd1828.in0! proc.memory.rssMB 11260.40625 disk.drbd1356.in0 disk.drbd2111.out0 disk.drbd1817.out0 disk.drbd1550.in0 disk.sda.out 159.89453125 disk.drbd1510.in0 disk.drbd1356.out0 disk.drbd1673.in0 disk.rbd0.out0* proc.memory.percent17.182016372680664 disk.sda2.in0 disk.drbd2041.out0 disk.drbd1938.in0 cpu13.74744077619587 proc.cpu.threads29 network.recv692585 disk.drbd1226.out0 disk.drbd1552.in0 disk.dm-0.in 4.64453125 disk.drbd1584.out0# disk.nvme0n1.out268.5673828125 disk.drbd1931.out0 disk.rbd0.in0 disk.drbd2118.out0 disk.drbd2111.in0 disk.drbd1673.out0 disk.drbd1938.out0$ memory_percent41.25276207923889 disk.drbd2118.in0 disk.dm-0.out268.5673828125* proc.memory.availableMB38500.58984375 network.sent 1.2228515e+07 disk.sda1.in0 disk.nvme0n1.in 4.64453125 disk.drbd1552.out0 disk.drbd1828.out0 disk.drbd1550.out0 disk.drbd1289.in0 disk.sda1.out0 disk.drbd1285.in0 disk.drbd1226.in0 disk.drbd2078.out0 disk.sda.in0 disk.drbd1285.out0 disk.drbd1510.out0 disk.sda2.out 159.89453125 disk.drbd1817.in0 disk.rbd1.in0 disk.drbd1965.in0 disk.drbd1965.out0 disk.drbd1289.out0 disk.drbd1584.in0 disk.drbd2041.in0^|2!- _wandbruntime5923  _runtime5923tD! & lr-AdamW/pg10.0003440329173923511 & lr-AdamW/pg20.0003440329173923511 trainer/global_step13349 % _timestamp1.7858012105203576e+09 _runtime5923.343319473 _step1064'n<]! trainer/global_step13349 % _timestamp1.7858012105203576e+09 _runtime5923.343319473 _step1064 & lr-AdamW/pg10.0003440329173923511 & lr-AdamW/pg20.0003440329173923511cY2!- _wandbruntime5923  _runtime5923?'!O \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated88688629 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated52561347 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 _runtime 5923.3446102 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 140982839 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 229671468 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated229180828836864 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated240927480776704 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11746564510208 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated11746651939840 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 140973183 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed88688627 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 140982839 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed11717816633056 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current233 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12064521213440 _step1065 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed240927376307712 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated90823529 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current231 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated235203764625012 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 140973183 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed241785224154112 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed235203661782528 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed223485845149472 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12064390539264 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current233 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12064521213440 _step1065 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed240927376307712 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated90823529 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current231 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated235203764625012 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 140973183 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed241785224154112 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed235203661782528 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed223485845149472 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12064390539264 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated88688629 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated52561347 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 _runtime 5923.3446102 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 140982839 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 229671468 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated229180828836864 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated240927480776704 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11746564510208 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated11746651939840 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 140973183 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed88688627 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 140982839 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed11717816633056 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392H2!- _wandbruntime5923  _runtime5923&!M S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed240945422690304 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11747444129792 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed88695270 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed229197978560512 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current252 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12065427719168 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed235221279428070 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _step1066 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 229688648 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 140983706 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated240945527166464 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed90829994 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated90830248 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed52565282 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated229738065635328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 140993376 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated88695272 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated229197995599872 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed223502585321792 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated88695272 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated11747531566592 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated52565284 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12065297052160 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated241803493354496 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed11747444129792 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated38264964 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 % _timestamp1.7858012108956828e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed38264712 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 _runtime5923.719047038 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed241803337783808 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed88695270 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 229688648 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated240945527166464 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 229678976 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640I&!M E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed240945422690304 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11747444129792 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed88695270 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed229197978560512 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current252 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12065427719168 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed235221279428070 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _step1066 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 229688648 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 140983706 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated240945527166464 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed90829994 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated90830248 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed52565282 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated229738065635328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 140993376 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated88695272 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated229197995599872 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed223502585321792 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated88695272 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated11747531566592 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated52565284 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12065297052160 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated241803493354496 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed11747444129792 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated38264964 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 % _timestamp1.7858012108956828e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed38264712 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 _runtime5923.719047038 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed241803337783808 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed88695270 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 229688648 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated240945527166464 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 229678976 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current1214k2!- _wandbruntime5923  _runtime5923aY ! .loss/FRt2/detection_loss2.341021712248524 1loss/SRqqt1/detection_loss1.3331839253302091 /loss/FBt1/detection_loss1.3205874963849782 2loss/SRqqt1/assignment_loss0.7483161352753925 2loss/SRqqt2/assignment_loss0.9361571513923599 2loss/FBt1/assignment_loss0.004012820884408086 3loss/FBt2/assignment_loss0.0012238021699735313 /loss/FBt2/detection_loss1.5327179280575365 epoch0 % _timestamp1.7858012108975992e+09 _runtime5923.720444245 0loss/FRt1/assignment_loss2.5813716406171974 /loss/FRt1/detection_loss2.5007160251790825 1loss/SRqqt2/detection_loss0.9742795826677613 &loss/total_loss16.283171377049875 trainer/global_step13349 _step1067 /loss/FRt2/assignment_loss2.009583156842452{5V! 0loss/FRt1/assignment_loss2.5813716406171974 /loss/FRt1/detection_loss2.5007160251790825 1loss/SRqqt2/detection_loss0.9742795826677613 &loss/total_loss16.283171377049875 trainer/global_step13349 _step1067 /loss/FRt2/assignment_loss2.009583156842452 .loss/FRt2/detection_loss2.341021712248524 1loss/SRqqt1/detection_loss1.3331839253302091 /loss/FBt1/detection_loss1.3205874963849782 2loss/SRqqt1/assignment_loss0.7483161352753925 2loss/SRqqt2/assignment_loss0.9361571513923599 2loss/FBt1/assignment_loss0.004012820884408086 3loss/FBt2/assignment_loss0.0012238021699735313 /loss/FBt2/detection_loss1.5327179280575365 epoch0 % _timestamp1.7858012108975992e+09 _runtime5923.720444245DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 % _timestamp1.7858012369462948e+09 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed241829710593536 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current254 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated242690927768064 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed241829710593536 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated230581214358528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated11790648075264 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated236084661704520 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated89020779 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 230531620 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 _runtime5949.769806911 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated91164358 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed52758195 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed230581189454848 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated230039166987264 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed89020777 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated241829815062528 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed242690774287360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current256 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated38406161 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 141510841 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated52758197 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed230039149947904 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed11761705096564 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated11790648075264 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11790560645632 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 141501185 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 % _timestamp1.7858012369462948e+09 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed241829710593536 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current254 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated242690927768064 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed241829710593536 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated230581214358528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated11790648075264 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated236084661704520 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated89020779 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 230531620 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 _runtime5949.769806911 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated91164358 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed52758195 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed230581189454848 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated230039166987264 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed89020777 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated241829815062528 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed242690774287360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current256 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated38406161 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 141510841 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated52758197 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed230039149947904 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed11761705096564 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated11790648075264 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11790560645632 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 141501185 JDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11762582569786 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated89027422 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 trainer/global_step13399 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 230548800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _step1070 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12110481703424 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed91170922 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed230598396571648 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed241847756976128 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed224339593937792 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated241847861452288 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 141511708 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated224339610977152 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated236102279350174 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 141521378 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated52762134 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current252 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856.E!M FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12110481703424 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed91170922 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed230598396571648 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed241847756976128 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed224339593937792 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated241847861452288 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 141511708 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated224339610977152 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated236102279350174 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 141521378 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated52762134 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current252 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated230056333750272 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11762582569786 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated89027422 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 trainer/global_step13399 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 230548800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _step1070]2!- _wandbruntime5950  _runtime5950g: ! epoch0 % _timestamp1.7858012373733811e+09 -loss/FBt2/detection_loss1.79691406066219 0loss/FRt2/assignment_loss1.9758125536897209 2loss/SRqqt1/assignment_loss0.8828904513269663 .loss/FRt2/detection_loss2.198529580838225 trainer/global_step13399 0loss/FRt1/assignment_loss2.7223321771935414 1loss/SRqqt2/detection_loss0.9518238084730566 _runtime5950.196301585 &loss/total_loss16.037518276678128 2loss/SRqqt2/assignment_loss0.8834356422094798 4loss/FBt1/assignment_loss2.0188303538713715e-06 /loss/FRt1/detection_loss2.5770789008391533 1loss/SRqqt1/detection_loss1.3314123015850783 /loss/FBt1/detection_loss0.7172720000568417 _step1071 3loss/FBt2/assignment_loss1.478097352058209e-05-o-! 3loss/FBt2/assignment_loss1.478097352058209e-05 epoch0 % _timestamp1.7858012373733811e+09 -loss/FBt2/detection_loss1.79691406066219 0loss/FRt2/assignment_loss1.9758125536897209 2loss/SRqqt1/assignment_loss0.8828904513269663 .loss/FRt2/detection_loss2.198529580838225 trainer/global_step13399 0loss/FRt1/assignment_loss2.7223321771935414 1loss/SRqqt2/detection_loss0.9518238084730566 _runtime5950.196301585 &loss/total_loss16.037518276678128 2loss/SRqqt2/assignment_loss0.8834356422094798 4loss/FBt1/assignment_loss2.0188303538713715e-06 /loss/FRt1/detection_loss2.5770789008391533 1loss/SRqqt1/detection_loss1.3314123015850783 /loss/FBt1/detection_loss0.7172720000568417 _step1071r;!(: U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts101.529' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent33.843 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0~V!(:  U disk.drbd1938.in0 disk.drbd1285.in0 disk.rbd1.in0 disk.drbd2111.in0 disk.rbd1.out 23.88671875 disk.drbd1584.in0 disk.drbd2118.out0) proc.memory.availableMB 38499.9140625 disk.drbd1552.in0 disk.rbd0.in0 disk.sda1.in0 disk.drbd1931.out0 network.sent 1.2321675e+07$ disk./.usageGB578.3524780273438 disk.drbd1289.out0 disk.drbd1510.out0 disk.drbd1356.out0! disk.dm-0.out269.32080078125 disk.drbd1817.in0 disk.drbd1965.out0 disk.drbd1931.in0 proc.cpu.threads29 disk.drbd1510.in0 disk.drbd2041.out0! proc.memory.rssMB 11260.40625 disk.drbd1226.out0 disk.dm-0.in 4.64453125 disk.drbd1817.out0% memory_percent41.253793239593506 disk.drbd1289.in0 disk.drbd1938.out0$ disk.nvme0n1.out269.32080078125 disk.drbd1673.out0 disk.drbd2041.in0 disk.rbd0.out0 disk.drbd1356.in0) disk./.usagePercent32.34449821847542 disk.drbd2078.out0 disk.sda2.in0 network.recv696610 disk.drbd1552.out0 disk.sda.in0 disk.drbd2118.in0 disk.nvme0n1.in 4.64453125 disk.drbd1285.out0 disk.drbd2078.in0 disk.drbd1584.out0 disk.sda2.out 160.828125 disk.drbd1550.in0 disk.drbd1226.in0 disk.drbd1673.in0* proc.memory.percent17.182016372680664 disk.drbd1828.in0 disk.drbd1828.out0 disk.drbd1550.out0 disk.sda.out 160.828125 disk.drbd1965.in0 disk.sda1.out0 disk.drbd2111.out0 cpu13.74542457119091iY!(:  U$ disk./.usageGB578.3525924682617 disk.drbd2118.in0 disk.sda1.out0 disk.drbd1584.in0 disk.drbd2111.out0 disk.rbd0.in0 network.sent 1.2366691e+07 disk.drbd1289.in0 disk.drbd1285.out0 disk.sda2.in0 disk.sda2.out 161.8203125 disk.rbd1.out 23.88671875 disk.drbd1828.out0 disk.sda.in0 disk.drbd1226.in0 disk.drbd2078.out0 disk.drbd1931.in0 disk.drbd1552.in0 disk.drbd2078.in0 disk.drbd1584.out0 disk.drbd1828.in0 disk.drbd1550.in0 disk.drbd1289.out0 disk.sda.out 161.8203125 disk.drbd1285.in0 disk.drbd1817.in0 disk.drbd1817.out0 disk.drbd1965.in0 disk.drbd1673.out0 disk.drbd1931.out0 cpu13.744753368484213 disk.drbd1510.in0* proc.memory.percent17.182016372680664$ disk.nvme0n1.out269.66845703125 disk.drbd1356.in0 disk.drbd2111.in0 disk.drbd1965.out0 disk.drbd1550.out0 disk.dm-0.in 4.64453125 disk.drbd1938.in0* proc.memory.availableMB38501.08984375 disk.drbd2041.in0 disk.nvme0n1.in 4.64453125! disk.dm-0.out269.66845703125$ memory_percent41.25199913978577) disk./.usagePercent32.34450461861066 disk.drbd1552.out0 proc.cpu.threads29 disk.drbd2118.out0 disk.drbd1673.in0 disk.drbd1356.out0 disk.drbd2041.out0 disk.rbd1.in0 disk.sda1.in0 disk.drbd1226.out0! proc.memory.rssMB 11260.40625 network.recv698071 disk.drbd1938.out0 disk.rbd0.out0 disk.drbd1510.out0ذ!(: V gpu.0.gpu13.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts107.078' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent35.69266666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0Rc2!- _wandbruntime5975  _runtime5975G! & lr-AdamW/pg10.0003466101360408817 & lr-AdamW/pg20.0003466101360408817 trainer/global_step13449 $ _timestamp1.785801262716258e+09 _runtime5975.539356855 _step1072S! & lr-AdamW/pg10.0003466101360408817 & lr-AdamW/pg20.0003466101360408817 trainer/global_step13449 $ _timestamp1.785801262716258e+09 _runtime5975.539356855 _step1072/z2!- _wandbruntime5975  _runtime5975'!O _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated11834644210688 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated91503947 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed89352927 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed236965455941544 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated11834644210688 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 trainer/global_step13449 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed91503694 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 231391772 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated225159879420832 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current251 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12154878664192 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 149286400 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11805679363196 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated89352929 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 142029187 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated243596448862720 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12154754281472 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed52955045 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed230897488098304 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated230897505137664 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed231441545294848 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 142038843 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed242732044879360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated231441570198528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 $ _timestamp1.785801262717514e+09 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated230897505137664 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11834556781056 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed89352927 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 124382720 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 231382114 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated242732149348352 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 _runtime5975.540946512 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated242732149348352X!O SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed89352927 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed236965455941544 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated11834644210688 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 trainer/global_step13449 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed91503694 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 231391772 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated225159879420832 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current251 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12154878664192 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 149286400 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11805679363196 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated89352929 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 142029187 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated243596448862720 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12154754281472 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed52955045 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed230897488098304 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated230897505137664 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed231441545294848 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 142038843 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed242732044879360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated231441570198528 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 $ _timestamp1.785801262717514e+09 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated230897505137664 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11834556781056 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed89352927 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 124382720 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 231382114 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated242732149348352 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 _runtime5975.540946512 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated242732149348352 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated11834644210688 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated9150394702!- _wandbruntime5975  _runtime5975'&!M KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 142039710 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated11835523837440 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated242750195738112 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated225176619593152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 _step1074 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 231399280 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 % _timestamp1.7858012631471474e+09 _runtime5975.970670057 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 231408952 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed11835436400640 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated89359572 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed89359570 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed230914654861312 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 142039710 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated236983176429682 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated38551734 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current257 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed91510459 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed52958982 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12155788577792 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11806556836530 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed242750091261952 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated243614565893120 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 142049380 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated89359572 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed89359570 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed230914654861312 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 231408952 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed38551477 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed243614406128128 trainer/global_step13449 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 231408952 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed38551477 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed243614406128128 trainer/global_step13449 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated11835523837440 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated242750195738112 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated225176619593152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 _step1074 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 231399280 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 % _timestamp1.7858012631471474e+09 _runtime5975.970670057 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 231408952 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed11835436400640 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated89359572 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed89359570 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed230914654861312 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 142039710 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated236983176429682 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated38551734 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current257 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed91510459 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed52958982 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12155788577792 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11806556836530 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed242750091261952 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated243614565893120 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 142049380 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated89359572 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed89359570 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed230914654861312 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated89685079 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated225996888036832 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 142566845 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed231755826248704 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated89685079 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated38693096 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated231755843288064 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed53151895 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12199934138368 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated243634483634176 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11849567826704 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated11878640346112 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated53151897 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 142566845 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11878552916480 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 [HDeBviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11878552916480 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12200064812544 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated244501990851072 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 _runtime6000.848686098 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 142557189 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated243634483634176 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed237846353021052 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated237846455863536 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed38692816 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated91844993 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed91844711 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 232251924 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current282 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed244501835273216 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed243634379165184 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 232242266 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 142557189 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed231755826248704 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated89685079 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated38693096 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated231755843288064 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed53151895 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12199934138368 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated243634483634176 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11849567826704 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated11878640346112 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated53151897 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 142566845 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11878552916480 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11878552916480 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12200064812544 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated244501990851072 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 _runtime6000.848686098 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 142557189 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated243634483634176 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed237846353021052 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated237846455863536 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed38692816 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated91844993 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed91844711 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 232251924 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current282 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed244501835273216 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed243634379165184 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 232242266 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 142557189 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated89685079 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated225996888036832 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 142566845 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 30219960322!- _wandbruntime6001  _runtime6001&!M F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11879432536064 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed232319108251648 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed89691720 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 _step1078 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed91851560 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12200968573952 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated53155834 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed237863970666594 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated11879519972864 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed38695728 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11850359496802 _runtime6001.219703019 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 232269104 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated89691722 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed11879432536064 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed243652425547776 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 232269104 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 142567712 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 232259432 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11850445300038 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated231773010051072 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed89691720 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated243652530023936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current267 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated91851829 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated243652530023936 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated89691722 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12200837906944 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated237864073509190 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 $ _timestamp1.785801288396324e+09 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 142577382 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated38695995 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated91851829 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated243652530023936 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated89691722 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12200837906944 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated237864073509190 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 $ _timestamp1.785801288396324e+09 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 142577382 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated38695995 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed91851560 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12200968573952 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated53155834 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed237863970666594 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated11879519972864 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed38695728 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11850359496802 _runtime6001.219703019 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 232269104 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated89691722 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed11879432536064 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed243652425547776 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 232269104 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 142567712 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 232259432 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11850445300038 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated231773010051072 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed89691720 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated243652530023936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current267 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated53348747 % _timestamp1.7858013106161776e+09 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 233102418 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated244536817920000 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed92185581 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed233162256974848 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 143094847 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed244536713451008 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated244536817920000 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current244 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 143094847 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated90017229 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed11893370487088 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed90017227 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed232614164399104 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated11922636481536 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11893456290212 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed244536713451008 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated232614181438464 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 233112076 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated38837080 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated233162281878528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 143085191 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12245154915328 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11922549051904 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed90017227 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated92185827 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed38836836 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 _runtime6023.439384053 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12245287686656 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed238727250100560 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed245407411890176 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated232614181438464 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated245407569565184 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11922549051904 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed53348745 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 J'!O LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated232614181438464 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated245407569565184 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11922549051904 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed53348745 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated53348747 % _timestamp1.7858013106161776e+09 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 233102418 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated244536817920000 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed92185581 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed233162256974848 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 143094847 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed244536713451008 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated244536817920000 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current244 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 143094847 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated90017229 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed11893370487088 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed90017227 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed232614164399104 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated11922636481536 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11893456290212 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed244536713451008 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated232614181438464 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 233112076 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated38837080 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated233162281878528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 143085191 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12245154915328 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11922549051904 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed90017227 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated92185827 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed38836836 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 _runtime6023.439384053 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12245287686656 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed238727250100560 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed245407411890176 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251d؇G2!- _wandbruntime6023  _runtime60238R!M BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed92192343 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed238744867746102 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated226850636825152 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 trainer/global_step13549 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed232631331162112 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed38839661 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11894247960310 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed232631331162112 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current232 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed90023870 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 143095714 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated232631348201472 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated90023872 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated92192575 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed233179464091648 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated238744970588698 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated244554864309760 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated53352684 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed244554759833600 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 233129256 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed90023870 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated11923516108288 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 143105384 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed245425529394688 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 % _timestamp1.7858013109884396e+09 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 _runtime6023.811815483 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11923428671488 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed226850619785792 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11894333763546v&!M PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 trainer/global_step13549 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed232631331162112 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed38839661 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11894247960310 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed232631331162112 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current232 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed90023870 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 143095714 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated232631348201472 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated90023872 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated92192575 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed233179464091648 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated238744970588698 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated244554864309760 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated53352684 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed244554759833600 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 233129256 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed90023870 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated11923516108288 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 143105384 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed245425529394688 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 % _timestamp1.7858013109884396e+09 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 _runtime6023.811815483 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11923428671488 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed226850619785792 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11894333763546 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed92192343 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed238744867746102 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated226850636825152 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712A,2!- _wandbruntime6023  _runtime60238O ! % _timestamp1.7858013109903169e+09 0loss/FRt2/assignment_loss2.4717283948909405 1loss/SRqqt1/assignment_loss1.194947216417763 2loss/FBt2/assignment_loss0.011899352569231882 0loss/SRqqt2/detection_loss1.081558000935364 _step1083 0loss/FRt1/assignment_loss2.4465151156516787 /loss/FBt2/detection_loss1.3274546314829163 _runtime6023.813354826 1loss/SRqqt2/assignment_loss0.808741228841925 /loss/FRt2/detection_loss2.4426094347307847 1loss/SRqqt1/detection_loss1.1033319059888047 epoch0 3loss/FBt1/assignment_loss0.0010150624911986483 /loss/FRt1/detection_loss2.3851001036769532 /loss/FBt1/detection_loss0.9151103406911716 %loss/total_loss16.19001078836873 trainer/global_step13549Uh! _runtime6023.813354826 1loss/SRqqt2/assignment_loss0.808741228841925 /loss/FRt2/detection_loss2.4426094347307847 1loss/SRqqt1/detection_loss1.1033319059888047 epoch0 3loss/FBt1/assignment_loss0.0010150624911986483 /loss/FRt1/detection_loss2.3851001036769532 /loss/FBt1/detection_loss0.9151103406911716 %loss/total_loss16.19001078836873 trainer/global_step13549 % _timestamp1.7858013109903169e+09 0loss/FRt2/assignment_loss2.4717283948909405 1loss/SRqqt1/assignment_loss1.194947216417763 2loss/FBt2/assignment_loss0.011899352569231882 0loss/SRqqt2/detection_loss1.081558000935364 _step1083 0loss/FRt1/assignment_loss2.4465151156516787 /loss/FBt2/detection_loss1.3274546314829163Єޛ!(: ֚U gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts126.536' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.17866666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0\!(:  U disk.rbd1.out 24.1611328125 disk.drbd1965.in0 disk.drbd1828.in0 disk.drbd1285.in0 network.sent 1.2462782e+07 disk.drbd2041.out0 disk.sda.out 163.67578125 disk.drbd1510.out0* proc.memory.availableMB38500.21484375 cpu13.743697738028343) disk./.usagePercent32.34450184521872 disk.drbd1938.out0 disk.drbd2078.in0 disk.nvme0n1.in 4.64453125 disk.drbd2111.in0 disk.drbd1817.in0 disk.drbd1931.in0 disk.drbd1552.in0 disk.drbd2118.in0 disk.sda.in0 disk.dm-0.out272.7080078125 disk.drbd1931.out0 disk.drbd1226.in0 network.recv702963 disk.sda2.out 163.67578125 disk.drbd2041.in0 disk.drbd1828.out0 disk.drbd1550.out0# disk.nvme0n1.out272.7080078125 disk.drbd1285.out0* proc.memory.percent17.182016372680664 disk.drbd1356.in0 disk.drbd1673.out0 disk.drbd1938.in0 disk.rbd0.in0! proc.memory.rssMB 11260.40625 disk.drbd2118.out0 disk.sda2.in0 disk.drbd1226.out0 disk.drbd2111.out0 disk.drbd1550.in0 disk.drbd1289.out0 disk.rbd0.out0 disk.dm-0.in 4.64453125$ disk./.usageGB578.3525428771973 disk.drbd1817.out0 disk.drbd1965.out0 disk.sda1.in0 disk.drbd1584.in0 disk.drbd1356.out0 disk.drbd1673.in0 disk.sda1.out0 disk.drbd1584.out0% memory_percent41.253334283828735 disk.rbd1.in0 disk.drbd1552.out0 disk.drbd1289.in0 disk.drbd1510.in0 disk.drbd2078.out0 proc.cpu.threads29 i!(: 贺U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts97.811' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent32.60366666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0^!(:  V disk.drbd1817.in0 disk.rbd1.in0 disk.drbd1289.out0 disk.dm-0.in 4.64453125 disk.nvme0n1.in 4.64453125 disk.drbd1356.out0 disk.drbd1226.in0% memory_percent41.252583265304565 disk.sda1.out0 disk.drbd1931.out0$ disk.nvme0n1.out273.02197265625 disk.drbd1965.out0 disk.drbd1828.in0 disk.sda.out 163.87109375 proc.cpu.threads29 disk.drbd2078.in0 disk.sda2.in0 disk.rbd1.out 24.2236328125 disk.drbd1552.in0 disk.drbd1550.in0 disk.sda1.in0 disk.drbd1356.in0 disk.drbd2111.in0 disk.drbd2111.out0 disk.drbd2041.out0 disk.drbd1289.in0 disk.rbd0.out0 disk.drbd1226.out0 network.sent 1.2507928e+07 network.recv704505* proc.memory.percent17.182016372680664 disk.drbd1285.out0 disk.sda2.out 163.87109375 disk.drbd1673.in0 disk.rbd0.in0* proc.memory.availableMB38500.70703125 disk.drbd1510.out0 disk.drbd2118.in0 disk.drbd1673.out0 disk.drbd2041.in0 disk.drbd1828.out0 disk.drbd1938.in0) disk./.usagePercent32.34450184521872 disk.drbd1285.in0 disk.drbd2118.out0 cpu13.743221005779482$ disk./.usageGB578.3525428771973 disk.drbd1965.in0 disk.drbd1938.out0 disk.drbd1931.in0 disk.drbd1550.out0 disk.drbd1510.in0 disk.drbd1584.in0 disk.drbd1817.out0! proc.memory.rssMB 11260.40625! disk.dm-0.out273.02197265625 disk.drbd2078.out0 disk.drbd1584.out0 disk.drbd1552.out0 disk.sda.in09v2!- _wandbruntime6045  _runtime6045! ' lr-AdamW/pg10.00035047596401367764 ' lr-AdamW/pg20.00035047596401367764 trainer/global_step13599 $ _timestamp1.785801332934858e+09 _runtime6045.757833915 _step1084d! ' lr-AdamW/pg10.00035047596401367764 ' lr-AdamW/pg20.00035047596401367764 trainer/global_step13599 $ _timestamp1.785801332934858e+09 _runtime6045.757833915 _step1084\22!- _wandbruntime6045  _runtime6045U٠'!O G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated233472519588864 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed227670888229472 _step1085 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated245439152205824 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated38979989 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed246312961892352 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11966545187328 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated246313113275904 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed234022612814848 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated245439152205824 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated233472519588864 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11966545187328 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 143622849 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed245439047736832 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11937344753720 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12290349077504 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated90349379 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed92525333 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed53545595 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current251 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed11937258950596 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 143613193 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 143613193 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated11966632616960 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated11966632616960 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated227670905268832 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 trainer/global_step13599 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated53545597 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12290475557376 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed90349377 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated234022637718528 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed38979738 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 % _timestamp1.7858013329357996e+09DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated233472519588864 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed11966545187328 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 143622849 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed245439047736832 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11937344753720 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12290349077504 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated90349379 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed92525333 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed53545595 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current251 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed11937258950596 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 143613193 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 143613193 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated11966632616960 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated11966632616960 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated227670905268832 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 trainer/global_step13599 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated53545597 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12290475557376 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed90349377 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated234022637718528 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed38979738 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 % _timestamp1.7858013329357996e+09 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated233472519588864 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed227670888229472 _step1085 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated245439152205824 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated38979989 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed246312961892352 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed11966545187328 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated246313113275904 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed234022612814848 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated245439152205824 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11967424806912 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 233989408 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _step1086 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 _runtime6046.139217609 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed53549532 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed11967424806912 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated38982839 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated239625867668206 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed227687628401792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed234039819931648 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11938136423818 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated11967512243712 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated53549534 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 233979736 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated90356022 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current263 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11938222227054 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated234039844835328 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated233489686351872 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated92532373 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current265 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated90356022 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 143623716 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed246331067297280 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed92532108 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 233989408 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12291384324096 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current263 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11938222227054 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated234039844835328 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated233489686351872 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated92532373 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current265 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated90356022 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 143623716 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed246331067297280 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed92532108 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 233989408 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12291384324096 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed11967424806912 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 233989408 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _step1086 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 _runtime6046.139217609 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed53549532 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed11967424806912 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated38982839 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated239625867668206 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed227687628401792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed234039819931648 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11938136423818 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated11967512243712 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated53549534 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 233979736 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated90356022 EDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 234832380 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed39122451 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed92864896 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated53742447 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed246341382022656 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11981233217228 _runtime6068.595535401 _step1089 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 144150851 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12010541322752 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed228507896845472 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed240489044259576 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 144150851 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed234882968654848 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 144141195 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed12010541322752 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 trainer/global_step13649 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated90681529 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 234822722 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed90681527 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current243 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated234882993558528 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed53742445 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed234330840699904 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed234330840699904 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated247218635828736 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated92865141 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed90681527 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed11981147414104 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12335642270208 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated228507913884832 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed246341382022656 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated246341486491648 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 % _timestamp1.7858013557722428e+09 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12010628752384 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated90681529 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated12010628752384 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed247218484445184 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current245 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548C6'"O _step1089 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 144150851 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12010541322752 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed228507896845472 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed240489044259576 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 144150851 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed234882968654848 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 144141195 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed12010541322752 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 trainer/global_step13649 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated90681529 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 234822722 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed90681527 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current243 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated234882993558528 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed53742445 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed234330840699904 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed234330840699904 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated247218635828736 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated92865141 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed90681527 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed11981147414104 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12335642270208 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated228507913884832 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed246341382022656 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated246341486491648 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 % _timestamp1.7858013557722428e+09 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12010628752384 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated90681529 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated12010628752384 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed247218484445184 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current245 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 234832380 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed39122451 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed92864896 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated53742447 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed246341382022656 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated11981233217228 _runtime6068.5955354015(2"- _wandbruntime6068  _runtime6068 K͟&"M LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated247236752056320 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed234900175771648 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 144161388 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 234849560 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated39125580 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated90688172 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated240506764747714 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed90688170 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed90688170 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed12011420942336 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12336420713984 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated228524654057152 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11982110690562 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed234348007462912 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12011508379136 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed39125330 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated234348024502272 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated53746384 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed228524637017792 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11982024887326 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12336551380992 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated92871964 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed53746382 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed92871712 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 144151718 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated234348024502272 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current252 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated234900200675328 % _timestamp1.7858013561422083e+09 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated12011508379136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 234839888 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime6068.965567492 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 _step1090 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated246359532881408 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 234849560 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 234839888 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed240506661905118 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 144161388 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed247236596485632 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856zIk"M REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated90688172 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 144151718 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 trainer/global_step13649 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated240506764747714 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed90688170 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed90688170 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed12011420942336 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12336420713984 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated228524654057152 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated11982110690562 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed234348007462912 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12011508379136 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed39125330 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated234348024502272 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated53746384 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed228524637017792 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed11982024887326 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12336551380992 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated92871964 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed53746382 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed92871712 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 144151718 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated234348024502272 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current252 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated234900200675328 % _timestamp1.7858013561422083e+09 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated12011508379136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 234839888 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime6068.965567492 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 _step1090 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated246359532881408 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 234849560 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 234839888 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed240506661905118 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 144161388 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed247236596485632 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated247236752056320 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed234900175771648 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 144161388 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 234849560 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated39125580 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated90688172 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed241369941339084 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 144678853 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 144669197 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed39264875 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12025121680736 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 235682874 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current255 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed229344905461472 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12054624887808 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated39265128 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed247243716308480 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated91013679 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated247243820777472 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated12054624887808 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed12054537458176 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated235743349398528 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed247243716308480 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12025035877612 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed241369941339084 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 144678853 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 144669197 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed39264875 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12025121680736 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 235682874 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current255 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed229344905461472 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12054624887808 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated39265128 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed247243716308480 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated91013679 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated247243820777472 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated12054624887808 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed12054537458176 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated235743349398528 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 JP  `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated235189195889664 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12380874625536 # _timestamp1.78580137767695e+09 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed91013677 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 235692532 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated229344922500832 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed247243716308480 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12025035877612 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 J6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated248142330773504 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 235700040 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed247261762691072 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated93211250 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated39268016 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12381641494016 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated229361662673152 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed235206345613312 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed53943232 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated235760556515328 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed229361645633792 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed12055417077760 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12025999154070 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 144689390 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed39267757 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _runtime6090.875223249 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated53943234 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed235760531611648 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed235206345613312 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed91020320 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated91020322 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed91020320 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 235709712 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12055504514560 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated247261867167232 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 235709712 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed93210989 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12381774258176 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 144689390 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 144679720 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed248142173105664 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed241387558984626 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 trainer/global_step13699 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed12025913350834 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12055417077760 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 #&"M PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated53943234 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed235760531611648 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed235206345613312 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed91020320 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated91020322 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed91020320 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 235709712 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12055504514560 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated247261867167232 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 235709712 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed93210989 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12381774258176 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 144689390 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 144679720 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed248142173105664 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed241387558984626 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 trainer/global_step13699 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed12025913350834 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12055417077760 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current261 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated235206362652672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated91020322 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated248142330773504 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 235700040 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed247261762691072 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated93211250 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated39268016 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12381641494016 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated229361662673152 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed235206345613312 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed53943232 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated235760556515328 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed229361645633792 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed12055417077760 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12025999154070 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 144689390 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed39267757 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _runtime6090.875223249ͩ2"- _wandbruntime6090  _runtime609025 " 2loss/SRqqt2/assignment_loss0.7673716264507389 .loss/FBt2/detection_loss1.174336509440433 /loss/FRt1/assignment_loss2.739433879547931 0loss/SRqqt2/detection_loss1.138829130503371 _runtime6090.876372596 _step1095 2loss/FBt2/assignment_loss0.056866188671950695 /loss/FBt1/detection_loss0.8676221727636522 epoch0 trainer/global_step13699 % _timestamp1.7858013780535464e+09 &loss/total_loss15.845834303148022 /loss/FRt2/assignment_loss2.249147696056585 2loss/SRqqt1/assignment_loss0.8791245630243046 2loss/FBt1/assignment_loss0.027082526165496227 /loss/FRt1/detection_loss2.3780706810190324 /loss/FRt2/detection_loss2.3940552344609953 1loss/SRqqt1/detection_loss1.1738940950435324 w6 " /loss/FRt2/detection_loss2.3940552344609953 1loss/SRqqt1/detection_loss1.1738940950435324 2loss/SRqqt2/assignment_loss0.7673716264507389 .loss/FBt2/detection_loss1.174336509440433 /loss/FRt1/assignment_loss2.739433879547931 0loss/SRqqt2/detection_loss1.138829130503371 _runtime6090.876372596 _step1095 2loss/FBt2/assignment_loss0.056866188671950695 /loss/FBt1/detection_loss0.8676221727636522 epoch0 trainer/global_step13699 % _timestamp1.7858013780535464e+09 &loss/total_loss15.845834303148022 /loss/FRt2/assignment_loss2.249147696056585 2loss/SRqqt1/assignment_loss0.8791245630243046 2loss/FBt1/assignment_loss0.027082526165496227 /loss/FRt1/detection_loss2.3780706810190324|BZ"(:  U disk.drbd1510.in0 disk.drbd1510.out0 disk.drbd1356.out0 disk.drbd1673.in0 disk.drbd1584.out0 disk.rbd1.out 24.5283203125 disk.drbd2041.out0 disk.sda.in0 disk.sda2.out 165.3671875 network.sent 1.2604402e+07 disk.sda1.out0 disk.drbd2118.in0 disk.drbd1817.out0 disk.drbd1931.in0 disk.drbd1938.out0 disk.dm-0.in 4.64453125 disk.drbd2078.out0 disk.drbd1584.in0 disk.drbd1356.in0 disk.drbd1965.in0 disk.drbd1552.in0 disk.drbd1828.in0 cpu13.741141906483062 proc.cpu.threads29 disk.drbd1931.out0 disk.rbd0.out0* proc.memory.percent17.182016372680664 disk.drbd1226.in0 disk.drbd1226.out0 disk.drbd1285.out0 disk.rbd1.in0 disk.drbd1552.out0 disk.drbd2078.in0 disk.drbd1817.in0 disk.drbd1938.in0 disk.sda1.in0 disk.drbd2041.in0 disk.drbd1828.out0 disk.drbd2111.out0 network.recv709948$ disk.nvme0n1.out274.24853515625$ memory_percent41.25243425369263 disk.drbd1550.in0 disk.drbd1550.out0 disk.drbd1289.out0 disk.sda.out 165.3671875* disk./.usagePercent32.344410109947155 disk.drbd2118.out0 disk.drbd2111.in0 disk.drbd1289.in0! disk.dm-0.out274.24853515625 disk.drbd1965.out0 disk.rbd0.in0) proc.memory.availableMB 38500.8046875# disk./.usageGB578.350902557373 disk.sda2.in0 disk.drbd1285.in0 disk.nvme0n1.in 4.64453125! proc.memory.rssMB 11260.40625 disk.drbd1673.out0DȈ"(: V gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts95.887' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent31.96233333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0c2"- _wandbruntime6113  _runtime6113Ul" % _timestamp1.7858014006784708e+09 _runtime6113.501464479 _step1096 & lr-AdamW/pg10.0003543417919864735 & lr-AdamW/pg20.0003543417919864735 trainer/global_step137490D" & lr-AdamW/pg20.0003543417919864735 trainer/global_step13749 % _timestamp1.7858014006784708e+09 _runtime6113.501464479 _step1096 & lr-AdamW/pg10.0003543417919864735S2"- _wandbruntime6113  _runtime6113 Տ"O RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed248146050594304 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12425965931008 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed93544505 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 145206855 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12426098702336 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated12098621023232 trainer/global_step13749 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12098621023232 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated236603705238528 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed91345827 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated236047534040064 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12069010144244 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated248146155063296 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed12098533593600 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current268 ZFDeviceStatsMonitor.oZQn_train_batch_start/inactive_split_bytes.all.freed249029646265856 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12098533593600 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed236047517000704 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 236543026 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated248146155063296 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 145206855 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated249029803940864 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 % _timestamp1.7858014006794937e+09 _step1097 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed236047517000704 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated230181931116832 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated242250941261076 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed248146050594304 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated236047534040064 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed91345827 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed242250838418592XO('"O RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed248146050594304 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12425965931008 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed93544505 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 145206855 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12426098702336 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated12098621023232 trainer/global_step13749 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12098621023232 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated236603705238528 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed91345827 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated236047534040064 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12069010144244 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated248146155063296 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed12098533593600 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current268 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed249029646265856 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12098533593600 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed236047517000704 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 236543026 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated248146155063296 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 145206855 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated249029803940864 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 % _timestamp1.7858014006794937e+09 _step1097 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed236047517000704 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated230181931116832 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated242250941261076 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed248146050594304 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated236047534040064 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed91345827 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed242250838418592q2"- _wandbruntime6113  _runtime6113&"M WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated249047913868800 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current249 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed12099413213184 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 236560192 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated93551567 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated248164201453056 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed91352470 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed236064683763712 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 236569864 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated54140084 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed230198654249792 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12069887617578 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 $ _timestamp1.785801401049163e+09 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated12099500649984 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 _step1098 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12099413213184 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated236620912355328 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed12069801814342 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 _runtime6113.872517962 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed54140082 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed248164096976896 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12426866652160 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed242268456064134 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed249047754103808 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12099500649984 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 trainer/global_step13749 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 145217392 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed39411236 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated242268558906730 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 145207722 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated91352472 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed236620887451648 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 145217392 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed93551318 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current247 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032| "M >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed39411236 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated242268558906730 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 145207722 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated91352472 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed236620887451648 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 145217392 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed93551318 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current247 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated249047913868800 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current249 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed12099413213184 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 236560192 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated93551567 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated248164201453056 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed91352470 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed236064683763712 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 236569864 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated54140084 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed230198654249792 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12069887617578 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 $ _timestamp1.785801401049163e+09 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated12099500649984 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 _step1098 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12099413213184 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated236620912355328 YObFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed12069801814342 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 _runtime6113.872517962 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed54140082 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed248164096976896 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12426866652160 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed242268456064134 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed249047754103808 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12099500649984 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 trainer/global_step13749 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 145217392 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1f2"- _wandbruntime6113  _runtime6113 " 3loss/FBt2/assignment_loss1.143647931511893e-05 1loss/SRqqt1/detection_loss1.1120086315866202 .loss/FBt1/detection_loss0.997860111585208 &loss/total_loss14.881848661590842 % _timestamp1.7858014010503678e+09 /loss/FRt2/assignment_loss2.025411396057575 /loss/FRt1/detection_loss2.2094483007025976 .loss/FRt2/detection_loss2.480455642299993 0loss/SRqqt2/detection_loss1.065686478302063 /loss/FBt2/detection_loss0.9409272733610123 epoch0 0loss/FRt1/assignment_loss2.3019527152020443 2loss/SRqqt1/assignment_loss0.8539223548803121 2loss/SRqqt2/assignment_loss0.8941642594741537 3loss/FBt1/assignment_loss6.165994991220189e-08 trainer/global_step13749 _runtime6113.873226621 _step1099r" 0loss/FRt1/assignment_loss2.3019527152020443 2loss/SRqqt1/assignment_loss0.8539223548803121 2loss/SRqqt2/assignment_loss0.8941642594741537 3loss/FBt1/assignment_loss6.165994991220189e-08 trainer/global_step13749 _runtime6113.873226621 _step1099 3loss/FBt2/assignment_loss1.143647931511893e-05 1loss/SRqqt1/detection_loss1.1120086315866202 .loss/FBt1/detection_loss0.997860111585208 &loss/total_loss14.881848661590842 % _timestamp1.7858014010503678e+09 /loss/FRt2/assignment_loss2.025411396057575 /loss/FRt1/detection_loss2.2094483007025976 .loss/FRt2/detection_loss2.480455642299993 0loss/SRqqt2/detection_loss1.065686478302063 /loss/FBt2/detection_loss0.9409272733610123 epoch0 7"(: U gpu.0.gpu27.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts95.682' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent31.894 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0L\"(:  εU network.sent 1.2649361e+07 disk.sda2.out 165.6328125 disk.drbd1550.in0 disk.drbd2118.in0 disk.drbd1817.in0! disk.dm-0.out274.51806640625 disk.drbd2118.out0 disk.drbd1828.in0* proc.memory.percent17.182016372680664 disk.sda1.out0 disk.drbd1510.out0 disk.drbd1226.out0$ disk.nvme0n1.out274.51806640625 disk.drbd1550.out0 proc.cpu.threads29 disk.dm-0.in 4.64453125 disk.drbd2111.in0 disk.drbd1673.in0 disk.drbd1828.out0 disk.drbd1938.out0 disk.rbd0.out0 disk.sda1.in0 disk.drbd2111.out0 disk.drbd1552.in0 cpu13.740597329462021 network.recv711486 disk.drbd1931.in0 disk.drbd1931.out0 disk.drbd2041.in0) proc.memory.availableMB 38499.9921875$ disk./.usageGB578.3508644104004 disk.drbd1584.out0 disk.nvme0n1.in 4.64453125 disk.drbd1226.in0 disk.drbd2041.out0 disk.drbd1965.in0 disk.drbd1965.out0 disk.rbd0.in0 disk.rbd1.out 24.6220703125 disk.drbd2078.out0 disk.drbd1584.in0 disk.drbd1356.in0 disk.drbd1552.out0 disk.drbd1289.out0 disk.sda.in0% memory_percent41.253674030303955 disk.sda.out 165.6328125 disk.drbd1285.out0 disk.drbd1817.out0 disk.drbd1289.in0! proc.memory.rssMB 11260.40625 disk.drbd1356.out0 disk.drbd1938.in0 disk.drbd1510.in0 disk.drbd2078.in0 disk.drbd1285.in0 disk.rbd1.in0* disk./.usagePercent32.344407976568746 disk.sda2.in0 disk.drbd1673.out0/C!"(: ‹U gpu.0.gpu35.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts124.523' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent41.507666666666665 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0o6_"(:  حU disk.sda1.in0 disk.sda2.out 166.14453125$ proc.memory.rssMB11260.41015625$ memory_percent41.25367999076843 disk.drbd1226.out0 disk.drbd1226.in0 disk.drbd1285.in0 disk.rbd1.out 24.6845703125 disk.drbd1965.out0 disk.drbd1673.out0 disk.drbd2078.in0 disk.drbd1584.out0) proc.memory.percent17.18202233314514 proc.cpu.threads29 disk.rbd0.out0 disk.drbd1817.in0 disk.drbd2041.in0 network.sent 1.2694487e+07 disk.drbd1938.in0 disk.rbd0.in0 disk.drbd1584.in0 disk.drbd2118.out0 disk.drbd1931.in0 disk.drbd1285.out0 disk.drbd2118.in0 disk.drbd1356.in0 disk.drbd1356.out0 disk.drbd2111.out0 disk.drbd1817.out0 disk.rbd1.in0 disk.drbd1673.in0* proc.memory.availableMB38499.98828125$ disk.nvme0n1.out274.78662109375) disk./.usagePercent32.34440370981193 disk.drbd1289.in0 disk.drbd1510.out0 disk.drbd1552.out0 cpu13.740075221303202 disk.drbd2041.out0 disk.drbd1828.in0 disk.drbd1289.out0 disk.dm-0.in 4.64453125 disk.drbd1965.in0 disk.drbd1828.out0 disk.drbd1550.in0 disk.drbd1550.out0 disk.drbd1938.out0 disk.sda.in0 disk.sda.out 166.14453125 disk.sda1.out0 disk.sda2.in0 network.recv713223 disk.drbd1510.in0 disk.drbd2078.out0 disk.nvme0n1.in 4.64453125 disk.drbd2111.in0 disk.drbd1931.out0$ disk./.usageGB578.3507881164551! disk.dm-0.out274.78662109375 disk.drbd1552.in0gP2"- _wandbruntime6136  _runtime6136#A" & lr-AdamW/pg10.0003556304013107389 & lr-AdamW/pg20.0003556304013107389 trainer/global_step13799 $ _timestamp1.785801423189033e+09 _runtime6136.012105864 _step1100-" & lr-AdamW/pg10.0003556304013107389 & lr-AdamW/pg20.0003556304013107389 trainer/global_step13799 $ _timestamp1.785801423189033e+09 _runtime6136.012105864 _step110032"- _wandbruntime6136  _runtime6136[y'"O QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 237403178 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12142617158656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 _runtime6136.016707528 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed236905855151104 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated12142617158656 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed249048384880128 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated91677979 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 % _timestamp1.7858014231901226e+09 _step1101 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed12142529729024 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated249048489349120 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current269 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12471301178880 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12112898607752 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated91677979 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 145734857 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 145725201 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed236905855151104 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed91677977 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current271 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed91677977 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated236905872190464 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 237412836 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated249048489349120 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12142529729024 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated39550728 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated236905872190464 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated54332997 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated243131838340584 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated231018939732832 trainer/global_step13799 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed54332995 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12112812804628 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed243131735498100 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed93883454 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed249048384880128 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 145734857 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 145725201 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12471170504704 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121d'"O G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated249935362257408 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed249935206679552 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated93883725 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed237464036174848 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed231018922693472 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed39550459 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated249048489349120 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12142529729024 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated39550728 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated236905872190464 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated54332997 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated243131838340584 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated231018939732832 trainer/global_step13799 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed54332995 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12112812804628 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed243131735498100 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed93883454 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed249048384880128 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 145734857 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 145725201 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12471170504704 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 237403178 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12142617158656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 _runtime6136.016707528 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed236905855151104 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated12142617158656 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed249048384880128 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated91677979 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 % _timestamp1.7858014231901226e+09 _step1101 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed12142529729024 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated249048489349120 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current269 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12471301178880 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12112898607752 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated91677979 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 145734857 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 145725201 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed236905855151104 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed91677977 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current271 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed91677977 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated236905872190464 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 237412836 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated243149455986238 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12143496785408 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed93890301 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12472075649536 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated91684622 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 145745394 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed249953318941184 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed237481243291648 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 237430016 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated39553622 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640b H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated93890556 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed91684620 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated236923038953472 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 145735724 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed249066431262720 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 $ _timestamp1.785801423558002e+09 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 237420344 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed236923021914112 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12143409348608 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed39553369 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 145735724 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current253 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed54336932 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12113776081086 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated249066535738880 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12472204219392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed91684620 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 145745394 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated91684622 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 trainer/global_step13799 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 _step1102 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed231035662865792 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed12113690277850 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 237430016 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated39553622 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated93890556 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed91684620 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated236923038953472 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 145735724 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed249066431262720 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 $ _timestamp1.785801423558002e+09 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 237420344 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed236923021914112 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12143409348608 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed39553369 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 145735724 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current253 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed54336932 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12113776081086 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated249066535738880 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12472204219392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed91684620 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 145745394 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated91684622 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 trainer/global_step13799 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 _step1102 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed231035662865792 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed12113690277850 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated243149455986238 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12143496785408 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed93890301 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12472075649536 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated91684622 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 145745394 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed249953318941184 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed237481243291648 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 PDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12516364189696 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated39693423 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 _runtime6158.538922466 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed244012632577608 _step1105 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 238272988 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 % _timestamp1.7858014457157338e+09 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed94223022 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12156787071260 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated238324416918528 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated92010129 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 146262859 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed249950719165952 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated244012735420092 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated237764210340864 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12186613294080 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 238263330 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated249950823634944 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed249950719165952 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 146253203 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated94223270 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated237764210340864 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed231855931309472 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated231855948348832 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0!v "O QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated94223270 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated237764210340864 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed231855931309472 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated231855948348832 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12516364189696 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated39693423 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 _runtime6158.538922466 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed244012632577608 _step1105 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 238272988 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 % _timestamp1.7858014457157338e+09 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed94223022 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12156787071260 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated238324416918528 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated92010129 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 146262859 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed249950719165952 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated244012735420092 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated237764210340864 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12186613294080 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 238263330 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated249950823634944 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed249950719165952 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 146253203Z:2"- _wandbruntime6158  _runtime6158D}N&"M =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated237781377103872 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 146263726 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed54533782 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed244030250223150 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12157664544594 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed237781360064512 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 _step1106 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated54533784 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 238280496 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed92016770 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated244030353065746 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed92016770 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 238290168 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated231872688521152 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 146263726 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed249968765548544 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12517402213376 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12517265254912 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 238290168 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed39695950 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 % _timestamp1.7858014460817313e+09 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated249968870024704 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed12187405484032 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated92016772 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated249968870024704 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated94229983 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed250858864386560 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated238341624035328 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed237781360064512 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12187405484032 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 238280496 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed94229732 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated250859026248704 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated12187492920832 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated39696199 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated237781377103872 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed12157578741358 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated92016772 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 146273396 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed231872671481792 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712^rj&"M H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12517402213376 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12517265254912 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 238290168 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed39695950 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 % _timestamp1.7858014460817313e+09 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated249968870024704 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed12187405484032 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated92016772 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated249968870024704 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated94229983 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed250858864386560 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated238341624035328 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed237781360064512 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12187405484032 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 238280496 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed94229732 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated250859026248704 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated12187492920832 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated39696199 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated237781377103872 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed12157578741358 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated92016772 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 146273396 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed231872671481792 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated237781377103872 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 146263726 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed54533782 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed244030250223150 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12157664544594 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed237781360064512 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 _step1106 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated54533784 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 238280496 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed92016770 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated244030353065746 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed92016770 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 238290168 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated231872688521152 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 146263726 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed249968765548544ƣ:2"- _wandbruntime6158  _runtime6158LD " 1loss/SRqqt1/detection_loss1.4360358836864562 trainer/global_step13849 0loss/FRt2/assignment_loss2.7605313512145497 .loss/FRt2/detection_loss2.847770345748448 &loss/total_loss15.972105230286342 2loss/SRqqt1/assignment_loss0.8806752884401041 2loss/SRqqt2/assignment_loss0.9952210087804275 4loss/FBt2/assignment_loss1.6784606842179528e-06 /loss/FBt1/detection_loss0.6781378018654762 /loss/FBt2/detection_loss0.6877633973956108 epoch0 _runtime 6158.9062962 0loss/FRt1/assignment_loss2.1786566103797362 4loss/FBt1/assignment_loss1.4228195625271895e-07 /loss/FRt1/detection_loss2.4177404818944184 1loss/SRqqt2/detection_loss1.0895712401384745 % _timestamp1.7858014460834446e+09 _step1107](" _runtime 6158.9062962 0loss/FRt1/assignment_loss2.1786566103797362 4loss/FBt1/assignment_loss1.4228195625271895e-07 /loss/FRt1/detection_loss2.4177404818944184 1loss/SRqqt2/detection_loss1.0895712401384745 % _timestamp1.7858014460834446e+09 _step1107 1loss/SRqqt1/detection_loss1.4360358836864562 trainer/global_step13849 0loss/FRt2/assignment_loss2.7605313512145497 .loss/FRt2/detection_loss2.847770345748448 &loss/total_loss15.972105230286342 2loss/SRqqt1/assignment_loss0.8806752884401041 2loss/SRqqt2/assignment_loss0.9952210087804275 4loss/FBt2/assignment_loss1.6784606842179528e-06 /loss/FBt1/detection_loss0.6781378018654762 /loss/FBt2/detection_loss0.6877633973956108 epoch0!("(: 焘U gpu.0.gpu24.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts99.857' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent33.28566666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0LJV"(:  U disk.drbd1584.in0 disk.drbd1817.in0 disk.drbd1673.out0 disk.drbd2041.out0 disk.drbd1550.out0! disk.nvme0n1.out 275.70703125 disk.drbd1828.out0 network.recv715895 disk.drbd1938.out0 disk.sda.out 166.75390625 disk.drbd1285.out0 disk.drbd1550.in0 disk.rbd0.in0 disk.dm-0.out 275.70703125 disk.nvme0n1.in 4.64453125 disk.drbd1356.in0 disk.sda2.out 166.75390625 disk.drbd2111.in0 disk.drbd2111.out0 network.sent 1.274258e+07 disk.drbd1938.in0 disk.drbd2118.in0 disk.drbd1356.out0 disk.rbd1.in0) disk./.usagePercent32.34440818990659$ disk./.usageGB578.3508682250977 disk.drbd1289.out0 disk.drbd2078.out0 disk.drbd2118.out0 disk.drbd1931.in0 disk.drbd1828.in0 disk.rbd0.out0 disk.sda1.out0 disk.drbd1552.out0 disk.dm-0.in 4.64453125 disk.drbd1965.out0 disk.drbd1552.in0 disk.drbd1226.in0 disk.drbd1673.in0 disk.drbd1289.in0 disk.drbd1510.out0 disk.drbd1226.out0 disk.drbd2078.in0) proc.memory.percent17.18202233314514 disk.sda.in0 disk.drbd1931.out0 disk.drbd2041.in0 disk.drbd1584.out0 disk.drbd1817.out0* proc.memory.availableMB38499.65234375$ proc.memory.rssMB11260.41015625 proc.cpu.threads29 disk.drbd1510.in0 disk.sda2.in0 disk.rbd1.out 24.716796875 disk.drbd1965.in0 cpu13.739200433063088 disk.drbd1285.in0# memory_percent41.2541925907135 disk.sda1.in0X^ "(: U gpu.0.gpu23.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts125.318' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent41.772666666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0u"(:  V network.sent 1.2787551e+07) proc.memory.percent17.18202233314514 disk.drbd1552.in0 disk.drbd1356.in0 disk.drbd1673.in0 disk.drbd2041.out0 disk.drbd1828.out0 disk.drbd1285.out0 disk.sda.in0 proc.cpu.threads29 disk.drbd2078.out0! disk.nvme0n1.out 275.91796875 disk.drbd2111.out0 disk.drbd1931.out0 disk.drbd1226.out0 disk.drbd2078.in0 disk.drbd1965.out0 disk.drbd1550.in0 network.recv717223 disk.drbd1817.in0 disk.rbd1.out 24.810546875 disk.drbd1289.out0 disk.dm-0.out 275.91796875 cpu13.738704505442925 disk.nvme0n1.in 4.64453125 disk.drbd1817.out0 disk.drbd1931.in0 disk.drbd1510.in0$ proc.memory.rssMB11260.41015625 disk.drbd1356.out0 disk.drbd1552.out0 disk.drbd1828.in0 disk.sda1.out0 disk.drbd1285.in0 disk.sda2.out 167.28125 disk.drbd1673.out0) proc.memory.availableMB 38499.6484375) disk./.usagePercent32.34440712321739 disk.drbd1584.out0 disk.drbd1965.in0 disk.drbd2041.in0 disk.drbd1550.out0 disk.drbd1289.in0 d isk.drbd1938.in0$ disk./.usageGB578.3508491516113 disk.drbd2111.in0 disk.drbd1938.out0 disk.rbd0.in0 disk.sda.out 167.28125 disk.sda1.in0 disk.drbd2118.out0 disk.sda2.in0 disk.dm-0.in 4.64453125 disk.drbd2118.in0 disk.rbd0.out0 disk.drbd1510.out0$ memory_percent41.25419855117798 disk.drbd1226.in0 disk.drbd1584.in0 disk.rbd1.in0[2"- _wandbruntime6181  _runtime6181" ' lr-AdamW/pg10.00035820761995926945 ' lr-AdamW/pg20.00035820761995926945 trainer/global_step13899 % _timestamp1.7858014684041731e+09 _runtime6181.227124176 _step11089OO" _step1108 ' lr-AdamW/pg10.00035820761995926945 ' lr-AdamW/pg20.00035820761995926945 trainer/global_step13899 % _timestamp1.7858014684041731e+09 _runtime6181.2271241764T2"- _wandbruntime6181  _runtime6181y'"O R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed250853053451776 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated94563233 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 % _timestamp1.7858014684051232e+09 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 146781205 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated239184772758528 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 trainer/global_step13899 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed92342277 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated238622548491264 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 _runtime6181.228310521 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed239184747854848 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed250853053451776 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated12230609429504 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed92342277 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed54726695 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12561698227200 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12230521999872 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated39836536 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 239133140 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated244893632499600 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12200589731644 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed238622531451904 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current257 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12561561261568 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated232692956964832 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current255 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed39836281 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated250853157920768 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12200675534768 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed244893529657116 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated238622548491264 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed251746309116416 _step1109 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 146790861 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated250853157920768 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 146790861 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed12230521999872 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed232692939925472 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated92342279 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated251746470985728 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 146790861 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed12230521999872 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed232692939925472 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated92342279 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated251746470985728 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed250853053451776 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated94563233 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 % _timestamp1.7858014684051232e+09 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 146781205 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated239184772758528 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 trainer/global_step13899 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed92342277 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated238622548491264 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 _runtime6181.228310521 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed239184747854848 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed250853053451776 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated12230609429504 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed92342277 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed54726695 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12561698227200 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12230521999872 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated39836536 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 239133140 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated244893632499600 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12200589731644 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed238622531451904 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current257 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12561561261568 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated232692956964832 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current255 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed39836281 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated250853157920768 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12200675534768 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed244893529657116 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated238622548491264 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed251746309116416 _step1109 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 146790861 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated250853157920768 JDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 239150320 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12562598490624 trainer/global_step13899 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed238639698214912 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated250871204310528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed12231401619456 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated238639715254272 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 239150320 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 239140648 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed12201467204866 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed250871099834368 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 _runtime6181.608006562 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed238639698214912 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated232709697137152 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed92348920 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated250871204310528 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed94569741 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated39839358 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated12231489056256 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12231489056256 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12231401619456 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated244911250145254 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed251764420698112 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 146791728 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed39839109 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed239201954971648 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 _step1110 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 239140648 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated239201979875328 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed232709680097792 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0g2"M SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 239150320 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 239140648 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed12201467204866 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 REDeviceStatsMonitor.o P]!n_train_batch_end/active_bytes.large_pool.current17039360 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed250871099834368 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 _runtime6181.608006562 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed238639698214912 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated232709697137152 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed92348920 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated250871204310528 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed94569741 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated39839358 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated12231489056256 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12231489056256 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12231401619456 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated244911250145254 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed251764420698112 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 146791728 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed39839109 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed239201954971648 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 _step1110 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 239140648 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated239201979875328 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed232709680097792 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated251764578365952 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 146801398 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed92348920 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 239150320 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12562598490624 trainer/global_step13899 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed238639698214912 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated250871204310528 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed12231401619456 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated238639715254272ښ2"- _wandbruntime6181  _runtime6181rɥ " .loss/FRt1/detection_loss2.804928813852481 1loss/SRqqt2/detection_loss1.0809274959363278 trainer/global_step13899 2loss/SRqqt2/assignment_loss0.7594921432245765 4loss/FBt2/assignment_loss1.7610534012785844e-05 /loss/FBt2/detection_loss1.1968207381044824 %loss/total_loss15.31018216548466 _runtime6181.609148702 _step1111 2loss/SRqqt1/assignment_loss0.9497378767901582 epoch0 % _timestamp1.7858014687863312e+09 3loss/FBt1/assignment_loss6.844854721158123e-07 .loss/FRt2/detection_loss2.227279764165481 1loss/SRqqt1/detection_loss1.2236440029830253 /loss/FBt1/detection_loss0.8754253787018599 /loss/FRt1/assignment_loss2.070794413338846 0loss/FRt2/assignment_loss2.1211132433679367F" /loss/FRt1/assignment_loss2.070794413338846 0loss/FRt2/assignment_loss2.1211132433679367 .loss/FRt1/detection_loss2.804928813852481 1loss/SRqqt2/detection_loss1.0809274959363278 trainer/global_step13899 2loss/SRqqt2/assignment_loss0.7594921432245765 4loss/FBt2/assignment_loss1.7610534012785844e-05 /loss/FBt2/detection_loss1.1968207381044824 %loss/total_loss15.31018216548466 _runtime6181.609148702 _step1111 2loss/SRqqt1/assignment_loss0.9497378767901582 epoch0 % _timestamp1.7858014687863312e+09 3loss/FBt1/assignment_loss6.844854721158123e-07 .loss/FRt2/detection_loss2.227279764165481 1loss/SRqqt1/detection_loss1.2236440029830253 /loss/FBt1/detection_loss0.8754253787018599}x>Y"(:  U$ disk./.usageGB578.3508491516113 disk.drbd2118.out0 disk.nvme0n1.in 4.64453125 disk.sda2.in0 disk.drbd1510.in0 disk.drbd1673.in0 disk.sda.out 167.640625 disk.drbd1285.in0 disk.sda2.out 167.640625 disk.rbd1.out 24.873046875 disk.drbd1673.out0 disk.drbd1931.in0 disk.drbd1552.in0 disk.drbd1828.out0 disk.drbd1965.in0 disk.drbd1828.in0 disk.drbd1289.in0 disk.drbd1938.in0 proc.cpu.threads29 disk.drbd2111.out0 disk.rbd0.in0 disk.sda.in0 disk.drbd1584.in0 network.sent 1.2790605e+07 disk.drbd1817.in0 disk.drbd1817.out0 disk.sda1.in0 disk.sda1.out0 disk.drbd1510.out0 disk.drbd1226.in0) proc.memory.percent17.18202233314514) disk./.usagePercent32.34440712321739$ disk.nvme0n1.out276.19482421875 disk.dm-0.in 4.64453125 disk.drbd2078.in0 disk.drbd2111.in0 disk.drbd2041.out0 disk.drbd1285.out0 cpu13.738314926833779 disk.drbd1938.out0$ proc.memory.rssMB11260.41015625 network.recv718317 disk.drbd1356.in0 disk.drbd1356.out0 disk.rbd1.in0 disk.drbd1552.out0) proc.memory.availableMB 38500.3359375 disk.drbd1584.out0 disk.drbd1931.out0 disk.drbd1550.in0 disk.drbd1289.out0 disk.rbd0.out0 disk.drbd1965.out0 disk.drbd2041.in0 disk.drbd2078.out0 disk.drbd2118.in0 disk.drbd1550.out0! disk.dm-0.out276.19482421875 disk.drbd1226.out0$ memory_percent41.25314950942993Ҍ"(: U gpu.0.gpu23.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts101.613' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent33.871 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0M2"- _wandbruntime6203  _runtime6203@>k" _step1112 ' lr-AdamW/pg10.00035949622928353476 ' lr-AdamW/pg20.00035949622928353476 trainer/global_step13949 % _timestamp1.7858014907821474e+09 _runtime6203.606704042o" ' lr-AdamW/pg10.00035949622928353476 ' lr-AdamW/pg20.00035949622928353476 trainer/global_step13949 % _timestamp1.7858014907821474e+09 _runtime6203.606704042 _step1112,02"- _wandbruntime6203  _runtime6203N'"O VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated251755492206592 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed239480869602304 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed245774426736624 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 239983634 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated39979558 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 147309207 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated239480886641664 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed12274518135296 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed251755387737600 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 trainer/global_step13949 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated92674429 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 % _timestamp1.7858014907828617e+09 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 239993292 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12606792295936 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated92674429 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12244478195152 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated239480886641664 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed39979304 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated240045128598528 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated12274605564928 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed92674427 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12274605564928 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed54923545 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current254 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed251755387737600 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current256 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12274518135296 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed252651895990784 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated252652051568640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed92674427 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12606922970112 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed94902849 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated233529965580832 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed239480869602304 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 147318863 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed240045103694848 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 _runtime6203.606932458 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed92674427 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12274605564928 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed54923545 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current254 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed251755387737600 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current256 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12274518135296 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed252651895990784 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated252652051568640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed92674427 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12606922970112 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed94902849 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated233529965580832 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed239480869602304 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 147318863 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed240045103694848 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 _runtime6203.606932458 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 PCDeviceStatsMoni tor.on_train_batch_start/active.large_pool.allocated92674429 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 % _timestamp1.7858014907828617e+09 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 239993292 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12606792295936 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121[2"- _wandbruntime6203  _runtime6203h &"M OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated92681072 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 147319730 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated39982405 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 240010472 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12607825986048 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated240062335715328 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed239498036365312 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 240000800 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12607691124736 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed245792044382166 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12275485191680 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated245792147224762 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated92681072 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12275397754880 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed252670001936384 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 147329400 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated12275485191680 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current240 _step1114 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 147319730 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated239498053404672 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current238 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated252670161701376 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 trainer/global_step13949 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 240000800 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed240062310811648 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12245441471610 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed12245355668374 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed239498036365312 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated54927484 % _timestamp1.7858014911585295e+09 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed39982167 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed251773434120192 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed94909649 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated251773538596352 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 240010472 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144sg&"M [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated239498053404672 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 147329400 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 _runtime 6203.98214325 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed233546688713792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed251773434120192 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed94909649 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated251773538596352 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 240010472 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated92681072 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 147319730 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated39982405 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 240010472 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12607825986048 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated240062335715328 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed239498036365312 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 240000800 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12607691124736 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed245792044382166 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12275485191680 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated245792147224762 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated92681072 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12275397754880 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed252670001936384 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 147329400 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated12275485191680 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current240 _step1114 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 147319730 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated239498053404672 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current238 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated252670161701376 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 trainer/global_step13949 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 240000800 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed240062310811648 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12245441471610 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed12245355668374 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed239498036365312 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated54927484 % _timestamp1.7858014911585295e+09 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed39982167 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 EDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed252657722023424 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed55120395 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated240339224792064 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12318601700352 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated234366974196832 _runtime6226.016645176 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated12318601700352 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed12318514270720 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 147846865 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated253557650214912 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed246655323816132 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed93006577 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated93006579 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12652039296512 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12288366658660 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12652165776384 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 126479872 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 240843786 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed95244423 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed240905459534848 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed234366957157472 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed240339207752704 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated95244699 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated240905484438528 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated246655426658616 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated240339224792064 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed93006577 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated252657826492416 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated93006579 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current274 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12318514270720 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed253557498831360 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 _step1117 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed240339207752704 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12288452461784 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 % _timestamp1.7858015131897645e+09 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12288452461784 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 % _timestamp1.7858015131897645e+09 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed252657722023424 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed55120395 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated240339224792064 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12318601700352 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated234366974196832 _runtime6226.016645176 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated12318601700352 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed12318514270720 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 147846865 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated253557650214912 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed246655323816132 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed93006577 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated93006579 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12652039296512 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12288366658660 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12652165776384 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 126479872 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 240843786 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed95244423 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed240905459534848 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed234366957157472 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed240339207752704 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated95244699 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated240905484438528 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated246655426658616 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated240339224792064 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed93006577 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121e9N2"- _wandbruntime6226  _runtime6226 &"M H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 240860952 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed240356374515712 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated253575764683264 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated246673044304270 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed40126897 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 147857402 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12319393890304 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated40127166 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 _runtime6226.403502842 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed240356374515712 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed55124332 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 147847732 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated252675872882176 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12652944558080 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed12289244131882 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current271 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _step1118 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12319481327104 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated12319481327104 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated95251500 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 240870624 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated240922691555328 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed234383697329792 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated240356391555072 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed93013220 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 240860952 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated234383714369152 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 240870624 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed95251229 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 trainer/global_step13999 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12653073127936 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated252675872882176 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed246672941461674 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed05pF "M J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed93013220 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed240922666651648 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 147857402 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 _runtime6226.403502842 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed240356374515712 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed55124332 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 147847732 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated252675872882176 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12652944558080 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed12289244131882 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current271 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _step1118 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12319481327104 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated12319481327104 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated95251500 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 240870624 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated240922691555328 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed234383697329792 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated240356391555072 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed93013220 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 240860952 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated234383714369152 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 240870624 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed95251229 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 trainer/global_step13999 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12653073127936 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated252675872882176 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed246672941461674 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 240860952 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed240356374515712 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated253575764683264 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated246673044304270 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0e REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed40126897 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 147857402 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12319393890304 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated40127166 PDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed95585820 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated241197562942464 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated241765840278528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 148365211 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed247536220895640 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated247536323738124 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed93338727 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12697213303296 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed253560056309248 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12362597835776 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed40268575 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed93338727 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12332255122168 trainer/global_step14049 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12362510406144 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed93338727 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12697213303296 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed253560056309248 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12362597835776 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed40268575 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed93338727 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12332255122168 trainer/global_step14049 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12362510406144 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed95585820 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated241197562942464 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated241765840278528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464"$2"- _wandbruntime6248  _runtime6248b"v#M OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12698118235136 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 241721104 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 % _timestamp1.7858015359932168e+09 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed241214712666112 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 241721104 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated247553941383778 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime6248.816638215 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 241730776 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 148375734 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated253578207168000 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 148375734 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated55321184 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12363477462528 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12363390025728 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated95592924 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed55321182 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed241214712666112 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed12363390025728 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed93345370 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current256 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed12333132595390 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated253578207168000 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 241730776 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated241214729705472 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated241214729705472 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12698248902144 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12333218398626 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 148385404 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed95592666 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated254481296297472 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed254481140726784 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12333218398626 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 148385404 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed95592666 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated254481296297472 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed254481140726784 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 241721104 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated247553941383778 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime6248.816638215 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 241730776 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 148375734 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated253578207168000 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 148375734 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated55321184 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12363477462528 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12363390025728 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated95592924 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed55321182 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed241214712666112 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed12363390025728 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed93345370 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current256 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed12333132595390 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated253578207168000 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 241730776 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated241214729705472 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated241214729705472 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0d#2#- _wandbruntime6248  _runtime6248~!h# 1loss/SRqqt1/detection_loss1.1039173534760873 1loss/SRqqt2/detection_loss0.9989334215702367 epoch0 3loss/FBt1/assignment_loss7.823104297699501e-08 .loss/FRt1/detection_loss2.677228099830887 % _timestamp1.7858015359949536e+09 _step1123 /loss/FBt2/detection_loss1.3808267438434996 trainer/global_step14049 _runtime6248.817785255 /loss/FRt2/assignment_loss2.335442414704491 2loss/SRqqt1/assignment_loss0.7685670996146897 .loss/FRt2/detection_loss2.424694777499227 /loss/FBt1/detection_loss0.9392946294392459 $loss/total_loss16.0787156361344 -loss/FRt1/assignment_loss2.4388248026371 2loss/SRqqt2/assignment_loss1.0109831606102495 4loss/FBt2/assignment_loss3.0546776437212486e-06@b # .loss/FRt2/detection_loss2.424694777499227 /loss/FBt1/detection_loss0.9392946294392459 $loss/total_loss16.0787156361344 -loss/FRt1/assignment_loss2.4388248026371 2loss/SRqqt2/assignment_loss1.0109831606102495 4loss/FBt2/assignment_loss3.0546776437212486e-06 1loss/SRqqt1/detection_loss1.1039173534760873 1loss/SRqqt2/detection_loss0.9989334215702367 epoch0 3loss/FBt1/assignment_loss7.823104297699501e-08 .loss/FRt1/detection_loss2.677228099830887 % _timestamp1.7858015359949536e+09 _step1123 /loss/FBt2/detection_loss1.3808267438434996 trainer/global_step14049 _runtime6248.817785255 /loss/FRt2/assignment_loss2.335442414704491 2loss/SRqqt1/assignment_loss0.7685670996146897gT#(:  U disk.drbd1817.out0 disk.sda1.in0 disk.drbd1285.out0$ memory_percent41.25547409057617 disk.drbd2118.in0 disk.drbd1965.in0) proc.memory.percent17.18202233314514 network.recv724712 disk.drbd1673.in0 disk.drbd1828.in0 disk.sda.out 169.296875 disk.drbd1285.in0 disk.drbd1510.out0 cpu13.736387450043546 disk.drbd1226.out0 disk.drbd1931.out0 disk.drbd1552.out0 disk.rbd0.out0 disk.sda.in0 disk.drbd1510.in0 proc.cpu.threads29 disk.rbd1.in0 disk.dm-0.in 4.64453125 disk.sda2.out 169.296875 disk.drbd1828.out0& proc.memory.availableMB 38498.8125 disk.sda2.in0 disk.drbd2118.out0 disk.drbd1356.in0 disk.drbd1817.in0 disk.drbd1552.in0 disk.drbd1289.out0 disk.drbd1550.in0 disk.drbd2078.in0 network.sent 1.2929217e+07 disk.dm-0.out 277.166015625 disk.drbd1226.in0" disk.nvme0n1.out 277.166015625 disk.drbd1965.out0 disk.drbd1938.out0 disk.drbd1673.out0 disk.drbd2041.out0 disk.drbd1938.in0* disk./.usagePercent32.344419923487834 disk.drbd2078.out0 disk.drbd1584.out0 disk.drbd2111.in0 disk.drbd1931.in0 disk.drbd2041.in0 disk.drbd1289.in0$ disk./.usageGB578.3510780334473$ proc.memory.rssMB11260.41015625 disk.drbd1356.out0 disk.rbd0.in0 disk.sda1.out0 disk.drbd1584.in0 disk.nvme0n1.in 4.64453125 disk.drbd2111.out0 disk.rbd1.out 25.0615234375 disk.drbd1550.out0K#(: V gpu.0.gpu0.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts95.653' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.884333333333338 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0TF#(: U gpu.0.gpu22.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts96.246' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent32.082 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0.H,Y#(:  V disk.drbd1817.in0 disk.drbd1285.in0 disk.drbd2118.out0 disk.drbd1931.out0( proc.memory.availableMB 38498.046875 disk.drbd1817.out0 disk.drbd2111.in0 cpu13.735886710457315 disk.rbd1.out 25.0615234375 disk.drbd1285.out0 proc.cpu.threads29 disk.drbd2118.in0 disk.drbd2078.in0 disk.drbd1552.out0 disk.drbd1289.out0 disk.sda.in0 disk.nvme0n1.in 4.64453125 disk.rbd1.in0) disk./.usagePercent32.34442312355545 disk.drbd1226.in0 disk.drbd1673.out0 disk.drbd1550.out0$ memory_percent41.25664234161377 disk.drbd2111.out0 disk.drbd2041.out0 disk.drbd1356.in0 disk.drbd2041.in0 disk.drbd1938.in0 disk.drbd1938.out0 disk.drbd1584.out0 disk.dm-0.out 277.357421875 disk.drbd1550.in0 network.recv726322 disk.drbd1226.out0" disk.nvme0n1.out 277.357421875 disk.drbd1356.out0 disk.drbd1289.in0 disk.sda.out 169.77734375) proc.memory.percent17.18202233314514 disk.drbd2078.out0 disk.drbd1965.in0 disk.rbd0.in0 disk.sda1.in0 disk.sda2.in0 disk.rbd0.out0 disk.sda1.out0 disk.drbd1510.out0 disk.drbd1828.in0 disk.drbd1510.in0 network.sent 1.2974232e+07 disk.dm-0.in 4.64453125 disk.sda2.out 169.77734375 disk.drbd1552.in0$ proc.memory.rssMB11260.41015625 disk.drbd1965.out0 disk.drbd1673.in0 disk.drbd1931.in0 disk.drbd1828.out0 disk.drbd1584.in0$ disk./.usageGB578.3511352539062~ 2#- _wandbruntime6271  _runtime62718/# & lr-AdamW/pg10.0003633620572563307 & lr-AdamW/pg20.0003633620572563307 trainer/global_step14099 % _timestamp1.7858015585724967e+09 _runtime6271.395504541 _step1124-ޭ# & lr-AdamW/pg10.0003633620572563307 & lr-AdamW/pg20.0003633620572563307 trainer/global_step14099 % _timestamp1.7858015585724967e+09 _runtime6271.395504541 _step1124$m2#- _wandbruntime6271  _runtime6271 t{'#O UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed255368599313920 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed236040974389472 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 _runtime 6271.39688592 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 148902869 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed12406506541568 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 _step1125 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 148893213 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated12406593971200 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated55514097 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed242626171214848 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12376143585676 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated242626196118528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed242055884053504 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current253 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed95926018 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed248417117975148 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12376229388800 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 148902869 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 242573748 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12742428099072 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 trainer/global_step14099 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed254462390595072 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated40412174 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current251 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12742562967552 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed242055884053504 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated242055901092864 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12406593971200 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12406506541568 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 % _timestamp1.7858015585735598e+09 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated242055901092864 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed40411923 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated254462495064064 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated95926271 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed55514095 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated236040991428832 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 242564090 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated93670879 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed93670877 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0Ļ #O G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated55514097 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed242626171214848 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12376143585676 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated242626196118528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed242055884053504 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current253 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed95926018 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 UAD JeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed248417117975148 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12376229388800 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 148902869 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 242573748 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12742428099072 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 trainer/global_step14099 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed254462390595072 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated40412174 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current251 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12742562967552 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed242055884053504 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated242055901092864 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12406593971200 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12406506541568 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 % _timestamp1.7858015585735598e+09 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated242055901092864 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed40411923 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated254462495064064 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated95926271 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed55514095 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated236040991428832 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 242564090 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated93670879 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed93670877 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed255368599313920 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed236040974389472 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 _runtime 6271.39688592 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 148902869 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed12406506541568 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 _step1125 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 148893213 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated12406593971200 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 148903736 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated12407473597952 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 148903736 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed95932830 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12743332367360 _runtime6271.770598767 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated93677522 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed55518032 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 242590928 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed242073050816512 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed93677520 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 148913406 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated95933088 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated254480541453824 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated242073067855872 trainer/global_step14099 % _timestamp1.7858015589472296e+09 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current258 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed40414798 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated248434838463286 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 242581256 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated40415054 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated242643403235328 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12407386161152 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed242073050816512 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 242590928 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated254480541453824 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed93677520 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated93677522 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12377106862134 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 242581256 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 148913406 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed242643378331648 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed248434735620690 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed254480436977664 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed255386710699008 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12743465131520+tG&#M MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed40414798 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated248434838463286 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 242581256 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated40415054 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated242643403235328 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12407386161152 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed242073050816512 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 242590928 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated254480541453824 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed93677520 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated93677522 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12377106862134 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 242581256 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 148913406 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed242643378331648 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed248434735620690 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed254480436977664 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed255386710699008 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12743465131520 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 148903736 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated12407473597952 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 148903736 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed95932830 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12743332367360 _runtime6271.770598767 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated93677522 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed55518032 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 242590928 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed242073050816512 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed93677520 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 148913406 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated95933088 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated254480541453824 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated242073067855872 trainer/global_step14099 % _timestamp1.7858015589472296e+09 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current258 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360i{S2#- _wandbruntime6271  _runtime6271# /loss/FRt1/detection_loss2.7605711327840203 /loss/FRt2/detection_loss2.4408374464655496 _runtime6271.771802842 1loss/SRqqt1/detection_loss1.4241962815380909 _step1127 /loss/FBt1/detection_loss0.8128981604501766 %loss/total_loss15.90466785774166 epoch0 1loss/SRqqt1/assignment_loss0.746909911353776 2loss/SRqqt2/assignment_loss1.0186946711880898 2loss/FBt2/assignment_loss0.036234136670827866 1loss/SRqqt2/detection_loss0.8635839205505191 trainer/global_step14099 /loss/FRt2/assignment_loss2.079377672758447 -loss/FBt2/detection_loss1.03190264692239 % _timestamp1.7858015589489725e+09 /loss/FRt1/assignment_loss2.689453742275499 3loss/FBt1/assignment_loss8.134784273014984e-06q1 # 1loss/SRqqt2/detection_loss0.8635839205505191 trainer/global_step14099 /loss/FRt2/assignment_loss2.079377672758447 -loss/FBt2/detection_loss1.03190264692239 % _timestamp1.7858015589489725e+09 /loss/FRt1/assignment_loss2.689453742275499 3loss/FBt1/assignment_loss8.134784273014984e-06 /loss/FRt1/detection_loss2.7605711327840203 /loss/FRt2/detection_loss2.4408374464655496 _runtime6271.771802842 1loss/SRqqt1/detection_loss1.4241962815380909 _step1127 /loss/FBt1/detection_loss0.8128981604501766 %loss/total_loss15.90466785774166 epoch0 1loss/SRqqt1/assignment_loss0.746909911353776 2loss/SRqqt2/assignment_loss1.0186946711880898 2loss/FBt2/assignment_loss0.036234136670827866PsU#(:  V disk.drbd1550.in0 disk.drbd1510.in0 disk.dm-0.in 4.64453125 disk.drbd2078.in0 disk.drbd1584.in0 disk.drbd2041.in0 disk.drbd1828.out0 disk.drbd1584.out0 disk.drbd2118.in0# memory_percent41.2555456161499 cpu13.735955794768186 network.recv727333 disk.sda.out 170.1953125 disk.sda1.in0 disk.drbd1817.in0 disk.rbd1.out 25.1796875 proc.cpu.threads29 disk.rbd0.in0 disk.drbd1510.out0 disk.dm-0.out278.6416015625 disk.drbd1552.in0 disk.drbd2078.out0 disk.drbd2118.out0# disk.nvme0n1.out278.6416015625 disk.sda2.in0 disk.drbd2111.in0 disk.drbd1289.in0 disk.drbd1938.out0 disk.drbd1226.out0 disk.drbd1931.out0 disk.drbd2041.out0 disk.drbd1828.in0 disk.drbd1550.out0 disk.rbd0.out0 disk.sda.in0 network.sent 1.297724e+07 disk.drbd1817.out0 disk.rbd1.in0 disk.drbd1965.out0$ disk./.usageGB578.3513374328613 disk.drbd1289.out0 disk.drbd1285.in0( proc.memory.availableMB 38498.765625 disk.drbd1931.in0$ proc.memory.rssMB11260.41015625* disk./.usagePercent32.344434430461014 disk.drbd1673.in0 disk.drbd1673.out0 disk.drbd1552.out0 disk.drbd1226.in0 disk.drbd1356.in0 disk.drbd1356.out0 disk.sda2.out 170.1953125 disk.drbd2111.out0 disk.drbd1965.in0 disk.sda1.out0) proc.memory.percent17.18202233314514 disk.drbd1285.out0 disk.drbd1938.in0 disk.nvme0n1.in 4.64453125#U #(: W gpu.0.gpu20.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts95.749' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent31.91633333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0k 2#- _wandbruntime6293  _runtime6293p# _step1128 ' lr-AdamW/pg10.00036465066658059595 ' lr-AdamW/pg20.00036465066658059595 trainer/global_step14149 % _timestamp1.7858015810153754e+09 _runtime6293.838310036DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 243424242 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated94003029 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 $ _timestamp1.785801581016295e+09 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 _step1129 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12450590106624 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 243433900 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated55710947 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed243486527054848 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 149430871 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 149430871 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated249298117897140 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated242914239243264 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated236878000044832 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 _runtime6293.839507691 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed242914222203904 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed255364724880896 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed256274113833984 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12420032049184 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed55710945 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated12450590106624 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated40556169 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed249298015054656 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current239 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated255364829349888 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12420117852308 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12787586779136 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 149421215 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated96267116 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated12450590106624 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated40556169 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed249298015054656 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current239 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated255364829349888 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12420117852308 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12787586779136 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 149421215 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated96267116 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 243424242 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated94003029 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 $ _timestamp1.785801581016295e+09 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 _step1129 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12450590106624 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 243433900 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated55710947 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed243486527054848 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 149430871 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 149430871 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated249298117897140 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated242914239243264 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated236878000044832 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 _runtime6293.839507691 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed242914222203904 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed255364724880896"&2#- _wandbruntime6294  _runtime6294] &#M C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated255382875739648 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12451382296576 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed236894723177792 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12420995325642 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12788629335040 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed242931388966912 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 243441408 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed94009670 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed12420909522406 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated249315735542794 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed96273721 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed40558839 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated94009672 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 149431738 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed242931388966912 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated255382875739648 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 149441408 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed249315632700198 _step1130 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 149431738 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated256292388410368 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 243451080 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 243441408 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12788496570880 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated12451469733376 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated242931406006272 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated236894740217152 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 149441408 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed55714882 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated96273972p #M WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated249315735542794 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed96273721 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed40558839 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated94009672 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 149431738 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed242931388966912 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated255382875739648 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 149441408 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed249315632700198 _step1130 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 149431738 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated256292388410368 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 243451080 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 243441408 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12788496570880 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated12451469733376 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated242931406006272 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated236894740217152 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 149441408 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed55714882 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated96273972 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated255382875739648 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12451382296576 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed236894723177792 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12420995325642 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12788629335040 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed242931388966912 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 243441408 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed94009670 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed12420909522406fD2#- _wandbruntime6294  _runtime6294# 1loss/SRqqt1/detection_loss1.4131814024465925 -loss/FBt1/detection_loss1.39167480146335 epoch0 % _timestamp1.7858015813974824e+09 0loss/FRt1/assignment_loss2.6489124298095703 0loss/SRqqt1/assignment_loss1.27478909298781 2loss/FBt1/assignment_loss0.017716380790827338 4loss/FBt2/assignment_loss3.3758854692678143e-06 /loss/FBt2/detection_loss1.3102604066580534 trainer/global_step14149 0loss/FRt2/assignment_loss2.4313349899492764 .loss/FRt2/detection_loss2.330633558528988 1loss/SRqqt2/detection_loss1.0909867337122559 &loss/total_loss17.332379028548466 _runtime6294.220323781 _step1131 2loss/SRqqt2/assignment_loss0.9075246442216448 .loss/FRt1/detection_loss2.515361212094625<= # 0loss/FRt1/assignment_loss2.6489124298095703 0loss/SRqqt1/assignment_loss1.27478909298781 2loss/FBt1/assignment_loss0.017716380790827338 4loss/FBt2/assignment_loss3.3758854692678143e-06 /loss/FBt2/detection_loss1.3102604066580534 trainer/global_step14149 0loss/FRt2/assignment_loss2.4313349899492764 .loss/FRt2/detection_loss2.330633558528988 1loss/SRqqt2/detection_loss1.0909867337122559 &loss/total_loss17.332379028548466 _runtime6294.220323781 _step1131 2loss/SRqqt2/assignment_loss0.9075246442216448 .loss/FRt1/detection_loss2.515361212094625 1loss/SRqqt1/detection_loss1.4131814024465925 -loss/FBt1/detection_loss1.39167480146335 epoch0 % _timestamp1.7858015813974824e+09*#(: U gpu.0.gpu38.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts96.566' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent32.18866666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0P$gW#(:  U disk.drbd1285.in0 disk.drbd1289.out0 disk.sda1.in0) disk./.usagePercent32.34443741719078 disk.drbd2078.in0 disk.drbd1584.out0 disk.drbd1289.in0 proc.cpu.threads29 network.recv728794 disk.drbd2078.out0 disk.drbd1817.in0 disk.drbd1285.out0$ memory_percent41.25640392303467$ disk.nvme0n1.out279.08837890625 disk.drbd2111.in0 disk.drbd2111.out0 disk.drbd1965.in0 disk.drbd2041.out0 disk.drbd1552.in0 disk.drbd1828.out0 disk.nvme0n1.in 4.64453125 disk.drbd1828.in0 disk.drbd1550.out0 disk.sda.out 170.7265625 disk.drbd1510.in0 cpu13.736444080702578 disk.drbd1356.in0 disk.rbd1.in0 disk.drbd1965.out0 disk.drbd1673.out0 disk.drbd1931.in0 disk.drbd1931.out0 disk.drbd2041.in0 disk.drbd2118.out0 disk.drbd1356.out0 disk.drbd1552.out0 disk.rbd0.in0 disk.sda.in0 disk.drbd1510.out0( proc.memory.availableMB 38498.203125 disk.drbd1550.in0 disk.drbd1226.in0 disk.drbd1673.in0 disk.drbd1584.in0 disk.sda2.out 170.7265625 disk.rbd0.out0! disk.dm-0.out279.08837890625 disk.drbd1938.out0 disk.sda1.out0 network.sent 1.3022189e+07 disk.drbd2118.in0 disk.drbd1817.out0 disk.rbd1.out 25.2734375) proc.memory.percent17.18202233314514 disk.dm-0.in 4.64453125 disk.sda2.in0 disk.drbd1938.in0 disk.drbd1226.out0$ proc.memory.rssMB11260.41015625# disk./.usageGB578.351390838623THZ#(:  U disk.dm-0.in 4.64453125 disk.drbd1550.in0 disk.sda.out 170.9921875 disk.sda1.in0 disk.sda1.out0 network.sent 1.3067319e+07 disk.drbd1510.in0 disk.drbd1584.in0 disk.nvme0n1.in 4.64453125$ disk.nvme0n1.out279.99267578125 disk.rbd1.in0 disk.drbd1931.out0$ disk./.usageGB578.3514099121094 disk.rbd0.in0 disk.drbd2111.in0* proc.memory.availableMB38498.69921875 disk.drbd1356.in0 disk.drbd1673.in0 disk.drbd1938.in0 disk.drbd1584.out0 disk.sda2.in0 disk.drbd2041.out0$ memory_percent41.25564694404602$ proc.memory.rssMB11260.41015625 disk.rbd0.out0 disk.drbd2078.in0 disk.drbd1356.out0) proc.memory.percent17.18202233314514 disk.drbd1673.out0 disk.drbd1828.out0 disk.drbd1289.in0 disk.drbd1285.in0 disk.drbd1285.out0! disk.dm-0.out279.99267578125 disk.sda2.out 170.9921875 cpu13.735968322440016 disk.drbd1965.in0 disk.drbd2041.in0 disk.drbd1552.out0 disk.drbd1938.out0 disk.drbd2118.in0 disk.drbd1931.in0 disk.drbd1550.out0 disk.drbd2111.out0 disk.drbd1817.out0 disk.drbd1226.out0 disk.drbd1289.out0 disk.drbd2078.out0 disk.drbd2118.out0 disk.drbd1817.in0 disk.drbd1965.out0 disk.drbd1510.out0 disk.drbd1226.in0 network.recv730507 disk.drbd1828.in0 disk.sda.in0 proc.cpu.threads29 disk.rbd1.out 25.3359375) disk./.usagePercent32.34443848387999 disk.drbd1552.in0S#(: ÃV gpu.0.gpu0.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts95.779' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.926333333333336 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors072#- _wandbruntime6316  _runtime6316-# trainer/global_step14199 $ _timestamp1.785801603785529e+09 _runtime6316.610697492 _step1132 ' lr-AdamW/pg10.00036593927590486126 ' lr-AdamW/pg20.000365939275904861263Y# trainer/global_step14199 $ _timestamp1.785801603785529e+09 _runtime6316.610697492 _step1132 ' lr-AdamW/pg10.00036593927590486126 ' lr-AdamW/pg20.000365939275904861266I2#- _wandbruntime6316  _runtime6316b'#O \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12494586242048 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12494498812416 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated243772577393664 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated40700086 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed244346882894848 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12464006315816 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated96607883 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current255 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed55907795 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current253 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed40699833 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed250178912134164 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated55907797 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated256267163635712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 149958873 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated94335179 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 244294052 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 244284394 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated256267163635712 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated250179014976648 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed94335177 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed256267059166720 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed237714991621472 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 149949217 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 _step1133 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed96607628 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated244346907798528 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated237715008660832 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12463920512692 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed94335177 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed256267059166720 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed257179610809344 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 _runtime6316.610777087 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated257179770581504 trainer/global_step14199 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated94335179 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024B+%#O MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed256267059166720 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed257179610809344 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 _runtime6316.610777087 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated257179770581504 trainer/global_step14199 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated94335179 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12494586242048 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12494498812416 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated243772577393664 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated40700086 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed244346882894848 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12464006315816 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated96607883 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current255 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed55907795 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current253 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed40699833 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed250178912134164 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated55907797 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated256267163635712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 149958873 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated94335179 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 244294052 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 244284394 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated256267163635712 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated250179014976648 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed94335177 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed256267059166720 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed237714991621472 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 149949217 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 _step1133 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed96607628 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated244346907798528 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 `LDeviceStatsMonitor.on_train_batch_start/reque)sted_bytes.large_pool.allocated237715008660832 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12463920512692 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed94335177 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0*S2#- _wandbruntime6316  _runtime6316l]x&#M trainer/global_step14199 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 244301560 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 244311232 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated94341822 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 149959740 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated40702821 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current222 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated94341822 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 % _timestamp1.7858016041511939e+09 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated12495465868800 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated55911734 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed40702601 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed256285105549312 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed243789727117312 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated237731748833152 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated243789744156672 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed94341820 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12495465868800 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed94341820 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated250196632622302 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _step1134 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated256285210025472 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12495378432000 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed257197725184512 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed243789727117312 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed12495378432000 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 149969410 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12833635172864 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 149959740 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated243789744156672 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated257197878658048 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed244364090011648 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 244311232 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed96614333 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated244364114915328 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed237731731793792 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 244301560 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 244311232 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated94341822 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 149959740 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated40702821 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current222 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated94341822 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 % _timestamp1.7858016041511939e+09 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated12495465868800 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated55911734 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed40702601 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed256285105549312 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed243789727117312 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated237731748833152 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated243789744156672 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed94341820 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12495465868800 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed94341820 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated250196632622302 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _step1134 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated256285210025472 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12495378432000 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed257197725184512 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed243789727117312 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed12495378432000 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 149969410 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12833635172864 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 149959740 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated243789744156672 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated257197878658048 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed244364090011648 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 244311232 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed96614333 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated244364114915328 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed2377317317937922e2#- _wandbruntime6316  _runtime6316ى# loss/FBt2/assignment_loss0 % _timestamp1.7858016041528473e+09 /loss/FRt2/detection_loss2.3323229811676254 epoch0 trainer/global_step14199 _runtime6316.975672911 _step1135 0loss/FRt1/assignment_loss2.5404965516768003 3loss/FBt1/assignment_loss5.617083196833499e-05 /loss/FRt1/detection_loss2.5097072620533014 1loss/SRqqt1/detection_loss1.5227041165726107 &loss/total_loss15.744286786194706 0loss/FRt2/assignment_loss2.1971993485649866 2loss/SRqqt1/assignment_loss0.8706120398338334 2loss/SRqqt2/assignment_loss0.9140953010258576 1loss/SRqqt2/detection_loss1.1287282569954793 /loss/FBt1/detection_loss0.5749742615673887 /loss/FBt2/detection_loss1.1533904959048544# 0loss/FRt1/assignment_loss2.5404965516768003 3loss/FBt1/assignment_loss5.617083196833499e-05 /loss/FRt1/detection_loss2.5097072620533014 1loss/SRqqt1/detection_loss1.5227041165726107 &loss/total_loss15.744286786194706 0loss/FRt2/assignment_loss2.1971993485649866 2loss/SRqqt1/assignment_loss0.8706120398338334 2loss/SRqqt2/assignment_loss0.9140953010258576 1loss/SRqqt2/detection_loss1.1287282569954793 /loss/FBt1/detection_loss0.5749742615673887 /loss/FBt2/detection_loss1.1533904959048544 loss/FBt2/assignment_loss0 % _timestamp1.7858016041528473e+09 /loss/FRt2/detection_loss2.3323229811676254 epoch0 trainer/global_step14199 _runtime6316.975672911 _step1135.h#(:  U disk.drbd1550.in0 disk.drbd1550.out0 disk.drbd1289.in0 disk.drbd2111.in0$ disk./.usageGB578.3514137268066 disk.drbd1552.in0 disk.drbd1289.out0 disk.drbd1584.out0$ disk.nvme0n1.out280.25439453125 disk.drbd1356.in0 disk.sda2.out 172.30078125 disk.drbd1965.in0 disk.drbd1828.out0 disk.rbd0.out0 disk.dm-0.in 4.64453125 disk.drbd1817.out0 disk.drbd1965.out0$ proc.memory.rssMB11260.41015625 disk.sda1.in0 disk.drbd1938.in0! disk.dm-0.out280.25439453125 disk.rbd1.in0 disk.drbd1285.out0) disk./.usagePercent32.34443869721783 disk.drbd1226.in0 network.recv731451 disk.drbd1673.in0 disk.drbd2041.out0 disk.rbd0.in0 disk.drbd2078.in0 disk.drbd1584.in0 disk.drbd2118.in0 network.sent 1.3070341e+07 disk.drbd1673.out0 disk.sda.out 172.30078125 disk.drbd1226.out0 disk.drbd2078.out0 proc.cpu.threads29 disk.drbd1510.in0 disk.drbd2111.out0$ memory_percent41.25569462776184 disk.drbd1552.out0 disk.drbd1356.out0* proc.memory.availableMB38498.66796875 disk.drbd1931.in0 disk.sda.in1.59375 disk.sda1.out0 disk.drbd1931.out0 disk.drbd2118.out0) proc.memory.percent17.18202233314514 disk.drbd1285.in0 disk.drbd1510.out0 disk.nvme0n1.in 4.64453125 disk.sda2.in1.59375 disk.drbd1817.in0 cpu13.735452353086353 disk.drbd1828.in0 disk.drbd2041.in0 disk.drbd1938.out0 disk.rbd1.out 25.3359375$f@#(: V gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts122.158' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent40.71933333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0P2#- _wandbruntime6339  _runtime63399B# $ _timestamp1.785801626203775e+09 _runtime6339.026775234 _step1136 ' lr-AdamW/pg10.00036722788522912663 ' lr-AdamW/pg20.00036722788522912663 trainer/global_step14249RZ# trainer/global_step14249 $ _timestamp1.785801626203775e+09 _runtime6339.026775234 _step1136 ' lr-AdamW/pg10.00036722788522912663 ' lr-AdamW/pg20.00036722788522912663caP.2#- _wandbruntime6339  _runtime6339 Ɗ?#O G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed257169393452544 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated94667329 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed40843102 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 trainer/global_step14249 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12538582377472 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12538494947840 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed12538494947840 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed94667327 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated40843374 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed94667327 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12878044310528 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current272 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated94667329 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 150486875 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed238552000237472 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated244630915544064 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 _step1137 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed251059809213672 _runtime6339.028136794 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 150477219 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current274 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed56104645 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated245207263638528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed244630898504704 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12877911539200 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 150486875 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12507894779324 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed244630898504704 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated244630915544064 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 JDeviceStatsMonitor.on_train_batch_start/oversize_segment]͡s.peak0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed258085150274048 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated12538582377472 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 150477219 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated238552017276832 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated257169497921536 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed257169393452544 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated94667329 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed40843102 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 trainer/global_step14249 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12538582377472 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12538494947840 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed12538494947840 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed94667327 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated40843374 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed94667327 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12878044310528 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current272 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated94667329 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 150486875 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed238552000237472 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated244630915544064 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 _step1137 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed251059809213672 _runtime6339.028136794 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 150477219 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current274 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed56104645 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated245207263638528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed244630898504704 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12877911539200 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 150486875 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12507894779324 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed244630898504704 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated244630915544064 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130d2#- _wandbruntime6339  _runtime6339h"&#M BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated96954701 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed244648065267712 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 150487742 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 245171384 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed96954443 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed56108582 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed257187439835136 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 245161712 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated94673972 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current258 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12878944100864 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 245171384 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated238568757449152 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed94673970 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 150487742 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed12539374567424 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current256 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated244648082307072 trainer/global_step14249 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated244648082307072 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 _runtime6339.401116127 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 150497412 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed40845861 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated251077529701810 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 _step1138 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated12539462004224 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed238568740409792 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12539374567424 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 120181248 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 $ _timestamp1.785801626577756e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed258103269771264 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated40846117 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12878823919616 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12539462004224 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed251077426859214 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated94673972 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200ũ&#M EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated244648082307072 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 _runtime6339.401116127 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 150497412 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed40845861 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated251077529701810 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 _step1138 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated12539462004224 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed238568740409792 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12539374567424 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 120181248 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 $ _timestamp1.785801626577756e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed258103269771264 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated40846117 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12878823919616 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12539462004224 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed251077426859214 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated94673972 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated96954701 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed244648065267712 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 150487742 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 245171384 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed96954443 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed56108582 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed257187439835136 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 245161712 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated94673972 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current258 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12878944100864 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 245171384 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated238568757449152 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed94673970 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 150487742 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed12539374567424 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current256 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated244648082307072 trainer/global_step14249b2#- _wandbruntime6339  _runtime6339(s# trainer/global_step14249 _step1139 2loss/FBt2/assignment_loss0.011307160722753905 /loss/FBt1/detection_loss1.1356447723694145 epoch0 2loss/SRqqt2/assignment_loss0.9702647107070406 4loss/FBt1/assignment_loss1.6271807901091506e -06 &loss/total_loss17.337368341076004 $ _timestamp1.785801626579422e+09 /loss/FRt1/assignment_loss2.867848088775856 /loss/FRt2/assignment_loss2.107422850480894 1loss/SRqqt1/assignment_loss1.086123681946278 /loss/FRt1/detection_loss2.7001037774742516 1loss/SRqqt1/detection_loss1.3517741743751146 /loss/FBt2/detection_loss1.7761532829143107 _runtime6339.402244248 .loss/FRt2/detection_loss2.367871610674916 1loss/SRqqt2/detection_loss0.9628526034543841# &loss/total_loss17.337368341076004 $ _timestamp1.785801626579422e+09 /loss/FRt1/assignment_loss2.867848088775856 /loss/FRt2/assignment_loss2.107422850480894 1loss/SRqqt1/assignment_loss1.086123681946278 /loss/FRt1/detection_loss2.7001037774742516 1loss/SRqqt1/detection_loss1.3517741743751146 /loss/FBt2/detection_loss1.7761532829143107 _runtime6339.402244248 .loss/FRt2/detection_loss2.367871610674916 1loss/SRqqt2/detection_loss0.9628526034543841 trainer/global_step14249 _step1139 2loss/FBt2/assignment_loss0.011307160722753905 /loss/FBt1/detection_loss1.1356447723694145 epoch0 2loss/SRqqt2/assignment_loss0.9702647107070406 4loss/FBt1/assignment_loss1.6271807901091506e-06 . i#(:  U disk.drbd1584.out0* proc.memory.availableMB38498.37890625 disk.drbd1356.in0 disk.drbd1965.out0 disk.sda.in1.59375 disk.drbd1584.in0 disk.drbd2041.in0 disk.drbd2078.out0 disk.drbd2118.in0 disk.sda2.out 172.7734375 disk.drbd1285.in0 disk.nvme0n1.in 4.64453125 disk.drbd2111.in0 disk.rbd1.in0 disk.rbd0.in0) proc.memory.percent17.18202233314514 disk.drbd2111.out0 disk.drbd1931.in0 disk.sda1.out0 disk.drbd2118.out0 disk.drbd1817.out0 disk.drbd1510.out0 proc.cpu.threads29) disk./.usagePercent32.34443869721783$ disk.nvme0n1.out280.56005859375 disk.drbd1356.out0 disk.drbd1965.in0 disk.drbd1285.out0 disk.sda2.in1.59375 disk.drbd1817.in0$ memory_percent41.25613570213318 disk.drbd1828.out0 disk.drbd1938.out0 disk.drbd1226.out0$ disk./.usageGB578.3514137268066 disk.drbd1552.in0 disk.drbd1828.in0 disk.drbd1289.in0 disk.drbd1226.in0 cpu13.735078565974863 disk.drbd1552.out0 disk.drbd1550.in0 disk.drbd1550.out0 disk.sda1.in0 network.sent 1.3115357e+07 disk.drbd1673.out0 disk.drbd1289.out0$ proc.memory.rssMB11260.41015625 disk.rbd1.out 25.3681640625 disk.drbd1673.in0 disk.drbd2041.out0 disk.sda.out 172.7734375 disk.drbd1510.in0 disk.drbd1931.out0 network.recv732886 disk.drbd1938.in0 disk.rbd0.out0 disk.dm-0.in 4.64453125! disk.dm-0.out280.56005859375 disk.drbd2078.in0gԜ#(: V gpu.0.gpu23.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts104.749' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent34.916333333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0ѻNk#(:  U disk.drbd1965.in0 disk.drbd2118.out0 disk.drbd2111.in0 disk.rbd1.out 25.4619140625 disk.drbd1965.out0 disk.drbd1673.in0 disk.drbd1931.out0 disk.drbd1828.in0 disk.rbd1.in0 disk.drbd2041.out0 disk.drbd1552.in0 disk.drbd1510.out0 disk.drbd1552.out0 disk.drbd1550.in0 disk.drbd1550.out0 disk.drbd2078.in0* disk./.usagePercent32.344437630528624 disk.drbd1289.in0 disk.rbd0.in0 disk.drbd1285.in0 disk.drbd2078.out0 disk.drbd1584.out0* proc.memory.availableMB38497.85546875$ disk./.usageGB578.3513946533203 disk.sda.in1.59375 disk.drbd1226.out0 disk.drbd1584.in0 network.sent 1.316034e+07 disk.drbd2118.in0 disk.sda2.in1.59375$ memory_percent41.25693440437317 disk.rbd0.out0$ proc.memory.rssMB11260.41015625) proc.memory.percent17.18202233314514 disk.drbd1931.in0 disk.drbd1938.out0 disk.sda1.in0! disk.dm-0.out280.77490234375 disk.drbd1817.in0 disk.drbd1828.out0 disk.sda2.out 173.03515625 disk.drbd1817.out0 cpu13.734568390192019 network.recv734243 disk.sda.out 173.03515625 disk.sda1.out0 disk.drbd1285.out0 disk.drbd1510.in0 disk.nvme0n1.in 4.64453125$ disk.nvme0n1.out280.77490234375 disk.drbd2041.in0 disk.drbd1289.out0 disk.drbd1938.in0 disk.drbd1356.in0 disk.drbd1356.out0 disk.drbd2111.out0 disk.drbd1673.out0 proc.cpu.threads29 disk.dm-0.in 4.64453125 disk.drbd1226.in04#(: 肕U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts127.792' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.59733333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0N2#- _wandbruntime6361  _runtime6361&:# & lr-AdamW/pg10.0003685164945533919 & lr-AdamW/pg20.0003685164945533919 trainer/global_step14299 % _timestamp1.7858016485613403e+09 _runtime6361.384303138 _step1140=:# _runtime6361.384303138 _step1140 & lr-AdamW/pg10.0003685164945533919 & lr-AdamW/pg20.0003685164945533919 trainer/global_step14299 % _timestamp1.7858016485613403e+09rU02#- _wandbruntime6361  _runtime63612i5'#O NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 151005221 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated251940809135664 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed245489236655104 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 124382720 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated258071832207360 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 151005221 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated97287341 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated40985844 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 % _timestamp1.7858016485622969e+09 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 trainer/global_step14299 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed94999477 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated239389025892832 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12551697439708 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed40985593 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed251940706293180 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 246004698 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 246014356 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed258990720451072 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12551783242832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 _runtime6361.385614511 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed94999477 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated258990869737472 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed12582491083264 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed97287088 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed246067594574848 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed258071727738368 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated94999479 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12582578512896 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated245489253694464 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12923250258944 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 _step1141 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2Λ'#O L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed94999477 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated239389025892832 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12551697439708 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed40985593 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed251940706293180 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 246004698 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 246014356 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed258990720451072 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12551783242832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 _runtime6361.385614511 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed94999477 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated258990869737472 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed12582491083264 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed97287088 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed246067594574848 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed258071727738368 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated94999479 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12582578512896 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated245489253694464 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12923250258944 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 _step1141 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 151005221 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated251940809135664 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed245489236655104 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 124382720 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated258071832207360 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 151005221 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated97287341 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated40985844 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 % _timestamp1.7858016485622969e+09 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 trainer/global_step14299"@z2#- _wandbruntime6361  _runtime6361́V#M REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed245506403418112 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated258089878597120 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed56305432 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 246031536 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated40988746 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated251958426781318 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated95006122 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated95006122 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed246084801691648 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated245506420457472 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated56305434 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 151025414 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current265 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12924161229312 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _step1142 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed97293913 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed259008828059648 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed95006120 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current267 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed251958323938722 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 % _timestamp1.7858016489297843e+09 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 246031536 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated239405766065152 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed258089774120960 _runtime 6361.75318475 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12924026368000 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated259008987824640 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed95006120 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 151015744 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed12552574912930|&#M EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _step1142 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed97293913 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed259008828059648 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed95006120 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current267 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed251958323938722 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 % _timestamp1.7858016489297843e+09 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 246031536 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated239405766065152 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed258089774120960 _runtime 6361.75318475 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12924026368000 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated259008987824640 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed95006120 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 151015744 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed12552574912930 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed245506403418112 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated258089878597120 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed56305432 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 246031536 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated40988746 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated251958426781318 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated95006122 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated95006122 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed246084801691648 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated245506420457472 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated56305434 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 151025414 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current265 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated12924161229312 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak2512#- _wandbruntime6361  _runtime6361P# /loss/FRt1/assignment_loss2.896964000618976 0loss/FRt2/assignment_loss1.8724653928778892 2loss/SRqqt2/assignment_loss0.9428735249312012 2loss/FBt1/assignment_loss4.23115360958678e-05 .loss/FRt1/detection_loss2.805782150099243 /loss/FRt2/detection_loss2.3412129648549613 1loss/SRqqt2/detection_loss1.2309615353089092 _runtime6361.754450047 1loss/SRqqt1/detection_loss1.0386059507237453 _step1143 2loss/SRqqt1/assignment_loss0.7366783455823009 epoch0 trainer/global_step14299 " _timestamp1.7858016489316e+09 4loss/FBt2/assignment_loss1.5041676834375867e-05 /loss/FBt1/detection_loss1.1512005926822038 /loss/FBt2/detection_loss1.0196519727488198 %loss/total_loss16.03645378364118 # 2loss/SRqqt1/assignment_loss0.7366783455823009 epoch0 trainer/global_step14299 " _timestamp1.7858016489316e+09 4loss/FBt2/assignment_loss1.5041676834375867e-05 /loss/FBt1/detection_loss1.1512005926822038 /loss/FBt2/detection_loss1.0196519727488198 %loss/total_loss16.03645378364118 /loss/FRt1/assignment_loss2.896964000618976 0loss/FRt2/assignment_loss1.8724653928778892 2loss/SRqqt2/assignment_loss0.9428735249312012 2loss/FBt1/assignment_loss4.23115360958678e-05 .loss/FRt1/detection_loss2.805782150099243 /loss/FRt2/detection_loss2.3412129648549613 1loss/SRqqt2/detection_loss1.2309615353089092 _runtime6361.754450047 1loss/SRqqt1/detection_loss1.0386059507237453 _step1143t,߄i#(:  U disk.drbd1828.out0) proc.memory.percent17.18202233314514 disk.nvme0n1.in 4.64453125 disk.drbd2111.out0 disk.sda1.out0 disk.dm-0.in 4.64453125 disk.drbd1226.in0 disk.drbd1584.in0 disk.drbd1931.in0 disk.drbd1356.out0 disk.drbd1226.out0 disk.drbd2078.in0 disk.sda.in1.59375 disk.drbd1817.in0 disk.drbd2041.in0 disk.drbd1550.in0 disk.drbd1356.in0 disk.sda2.in1.59375 disk.sda2.out 173.3671875 disk.drbd1289.out0 disk.drbd1938.out0 disk.drbd2078.out0( proc.memory.availableMB 38497.640625 disk.drbd1938.in0 disk.rbd0.out0 disk.sda.out 173.3671875 network.sent 1.3163423e+07 disk.drbd2118.out0 disk.drbd1552.out0 disk.drbd1289.in0 disk.rbd0.in0 disk.drbd1552.in0% memory_percent41.257262229919434 disk.drbd2118.in0 disk.drbd1965.in0 disk.drbd1673.out0! disk.dm-0.out281.04345703125 proc.cpu.threads29 disk.drbd2111.in0$ disk.nvme0n1.out281.04345703125 disk.drbd1965.out0 disk.sda1.in0$ proc.memory.rssMB11260.41015625 disk.rbd1.in0 disk.drbd2041.out0 disk.drbd1285.in0 disk.drbd1510.out0 network.recv735241* disk./.usagePercent32.344434430461014 disk.drbd1931.out0 disk.drbd1285.out0 disk.drbd1510.in0 disk.drbd1584.out0 disk.rbd1.out 25.5244140625 disk.drbd1828.in0 disk.drbd1550.out0 cpu13.734118867666588$ disk./.usageGB578.3513374328613 disk.drbd1817.out0 disk.drbd1673.in0#(: ęU gpu.0.gpu23.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts125.552' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent41.85066666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0˖U2#- _wandbruntime6384  _runtime63844}^# _runtime6384.046712841 _step1144 & lr-AdamW/pg10.0003698051038776572 & lr-AdamW/pg20.0003698051038776572 trainer/global_step14349 % _timestamp1.7858016712223566e+09,\p# & lr-AdamW/pg20.0003698051038776572 trainer/global_step14349 % _timestamp1.7858016712223566e+09 _runtime6384.046712841 _step1144 & lr-AdamW/pg10.0003698051038776572W2#- _wandbruntime6384  _runtime6384Ц+'#O M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 151533223 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 151533223 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12968257084928 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current239 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated259896365174784 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated252821706215172 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated240226034508832 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12626487218688 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed258974062024192 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 151542879 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed41129904 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated95331629 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed95331627 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12595585903216 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 _runtime6384.046807194 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed258974062024192 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated246347591844864 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 % _timestamp1.7858016712230556e+09 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed246347574805504 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 _step1145 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated95331629 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed240226017469472 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12595671706340 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated258974166493184 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed246347574805504 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current241 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12968389856256 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 246874508 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 151542879 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 2537553923}0a@#O G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 _step1145 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated95331629 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed240226017469472 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12595671706340 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated258974166493184 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed246347574805504 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current241 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated12968389856256 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 246874508 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 151542879 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 151533223 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 151533223 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed12968257084928 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current239 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated259896365174784 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated252821706215172 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated240226034508832 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12626487218688 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed258974062024192 WJDeviceStatsMonitor.on_train_batch_start/allocated_byteT6(Os.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 151542879 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed41129904 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated95331629 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed95331627 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12595585903216 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 _runtime6384.046807194 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed258974062024192 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated246347591844864 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 % _timestamp1.7858016712230556e+09 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed246347574805504 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 25375539212#- _wandbruntime6384  _runtime6384&#M ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated97635269 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 151543746 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated252839323860826 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated240242774681152 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12596549179674 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 246891688 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated95338272 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed246945157531648 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 246891688 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed240242757641792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed252839221018230 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12627454275072 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed95338270 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed259914314928128 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed97635014 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated246945182435328 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed41132732 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12969157396480 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0'4A&#M VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated56502284 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 % _timestamp1.7858016715937302e+09 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated258992212882944 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 246882016 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed97635014 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated246945182435328 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed41132732 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed12969157396480 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated97635269 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 151543746 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated252839323860826 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated240242774681152 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12596549179674 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 246891688 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated95338272 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed246945157531648 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 246891688 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed240242757641792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed252839221018230 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12627454275072 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed95338270 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed259914314928128 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed259876396310016 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed247788306254848 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 152061225 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed12670483354112 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13013437156352 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed253702500452196 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed95663777 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 trainer/global_step14399 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 152070881 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated12670570783744 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated259876500779008 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated260801894794752 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12639474366724 % _timestamp1.7858016935449104e+09 _step1149 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated95663779 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated97968703 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13013563636224 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 152070881 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121$&'#O ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated247205929995264 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 152070881 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated241063043124832 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed259876396310016 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated247205929995264 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 152061225 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current259 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated259876500779008 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated253702603294680 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed97968442 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed259876396310016 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed247788306254848 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 152061225 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed12670483354112 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13013437156352 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed253702500452196 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed95663777 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 trainer/global_step14399 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 152070881 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated12670570783744 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated259876500779008 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated260801894794752 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12639474366724 % _timestamp1.7858016935449104e+09 _step1149 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated95663779 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated97968703 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13013563636224 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 247751840 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated259894547168768 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed247805513371648 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed259894442692608 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed97975187 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 % _timestamp1.7858016939124844e+09 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated247223096758272 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 247751840 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated13014471993344 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed260819848406528 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 247742168 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated247805538275328 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12640437643182 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated241079783297152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed95670420 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated247223096758272 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated56699134 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed247223079718912 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 _runtime6406.735843941 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 152081418 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated259894547168768 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated253720220940334 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated97975460 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 _step1150 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 247742168 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated95670422 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated260820010268672 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13014335034880 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed56699132 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed95670420 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12671362973696 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 152081418 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated12671450410496 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated95670422 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed247223079718912 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated95670422 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed247223079718912 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 247751840 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated259894547168768 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed247805513371648 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed259894442692608 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed97975187 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 % _timestamp1.7858016939124844e+09 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated247223096758272 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 247751840 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated13014471993344 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed260819848406528 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 247742168 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak53852#- _wandbruntime6406  _runtime6406(!# 2loss/SRqqt2/assignment_loss0.8174743541886693 trainer/global_step14399 .loss/FBt2/detection_loss1.162697999466521 &loss/total_loss16.630317830294672 2loss/SRqqt1/assignment_loss0.9657280320360528 4loss/FBt2/assignment_loss0.00031145485502293013 /loss/FRt1/detection_loss2.4836971713117806 /loss/FRt2/detection_loss2.4992552793451717 /loss/FBt1/detection_loss1.4436847247804205 % _timestamp1.7858016939141276e+09 /loss/FRt1/assignment_loss2.453042375754161 loss/FBt1/assignment_loss0 epoch0 _runtime6406.736947846 _step1151 0loss/FRt2/assignment_loss2.3491428331895308 1loss/SRqqt1/detection_loss1.2781529276813755 0loss/SRqqt2/detection_loss1.177130677685963# loss/FBt1/assignment_loss0 epoch0 _runtime6406.736947846 _step1151 0loss/FRt2/assignment_loss2.3491428331895308 1loss/SRqqt1/detection_loss1.2781529276813755 0loss/SRqqt2/detection_loss1.177130677685963 2loss/SRqqt2/assignment_loss0.8174743541886693 trainer/global_step14399 .loss/FBt2/detection_loss1.162697999466521 &loss/total_loss16.630317830294672 2loss/SRqqt1/assignment_loss0.9657280320360528 4loss/FBt2/assignment_loss0.00031145485502293013 /loss/FRt1/detection_loss2.4836971713117806 /loss/FRt2/detection_loss2.4992552793451717 /loss/FBt1/detection_loss1.4436847247804205 % _timestamp1.7858016939141276e+09 /loss/FRt1/assignment_loss2.453042375754161 h#(:  U disk.sda2.in1.59375 disk.drbd1938.out0 disk.drbd1285.in0! disk.dm-0.out281.73583984375 disk.rbd1.out 25.8525390625 disk.drbd1965.out0 disk.drbd2041.in0 disk.drbd1285.out0* proc.memory.availableMB38498.00390625 disk.drbd1289.in0" proc.memory.rssMB 11260.296875 network.sent 1.3256641e+07 disk.drbd2118.in0 disk.drbd2111.in0 disk.rbd1.in0 disk.drbd1552.in0 disk.drbd1584.out0 disk.drbd2118.out0 disk.rbd0.in0 disk.drbd1510.in0$ memory_percent41.25670790672302 proc.cpu.threads29 disk.drbd1931.out0 disk.dm-0.in 4.64453125 disk.drbd2078.in0 disk.drbd2078.out0 disk.drbd1584.in0 disk.drbd1552.out0 disk.rbd0.out0 disk.drbd1510.out0 disk.drbd1226.in0 network.recv739404) disk./.usagePercent32.34439944305511 disk.drbd1931.in0 disk.drbd2041.out0 cpu13.733111515179148 disk.drbd1356.out0 disk.sda.out 174.4609375 disk.drbd1817.in0 disk.drbd1828.in0 disk.drbd1938.in0$ disk./.usageGB578.3507118225098 disk.drbd1356.in0 disk.drbd1817.out0 disk.sda.in1.59375 disk.sda1.out0 disk.drbd2111.out0 disk.drbd1965.in0 disk.drbd1673.in0 disk.drbd1673.out0 disk.sda1.in0* proc.memory.percent17.181849479675293 disk.drbd1828.out0 disk.drbd1550.out0 disk.nvme0n1.in 4.64453125$ disk.nvme0n1.out281.73583984375 disk.sda2.out 174.4609375 disk.drbd1550.in0 disk.drbd1226.out0 disk.drbd1289.out0'fq#(: ٞU gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts113.169' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent37.723 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0ɴ2#- _wandbruntime6428  _runtime6428# % _timestamp1.7858017158812962e+09 _runtime6428.704265958 _step1152 ' lr-AdamW/pg10.00037238232252618776 ' lr-AdamW/pg20.00037238232252618776 trainer/global_step14449B# % _timestamp1.7858017158812962e+09 _runtime6428.704265958 _step1152 ' lr-AdamW/pg10.00037238232252618776 ' lr-AdamW/pg20.00037238232252618776 trainer/global_step14449 P$2#- _wandbruntime6428  _runtime6428N'#O VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated260778835064832 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated248648686998528 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated254583500374188 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated56892047 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12714479489536 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 152598883 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12683362830232 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed260778730595840 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current264 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed98310217 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13058732962304 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated260778835064832 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated248064268145664 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13058598093824 % _timestamp1.7858017158822594e+09 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated95995929 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current266 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed241900034701472 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated98310483 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed95995927 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed260778730595840 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 248594812 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 152589227 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed248064251106304 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated261707419960832 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed56892045 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 152589227 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed248064251106304 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _step1153 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992N2'#O JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated95995929 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current266 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed241900034701472 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated98310483 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed95995927 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed260778730595840 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 248594812 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 152589227 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed248064251106304 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated261707419960832 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed56892045 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 152589227 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed248064251106304 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _step1153 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated260778835064832 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated248648686998528 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated254583500374188 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated56892047 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12714479489536 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 152598883 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12683362830232 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed260778730595840 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current264 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed98310217 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13058732962304 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated260778835064832 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated248064268145664 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13058598093824 % _timestamp1.7858017158822594e+09 JDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated96002572 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed96002570 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated260796881454592 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated248665894115328 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 248611992 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed98317076 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated56895984 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed261725373318656 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed12684240303454 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed12715359109120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed248081417869312 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed41421094 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated13059630579712 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed241916774873792 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13059504107008 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 152609420 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed248081417869312 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated254601118019842 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 248602320 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12715359109120 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12715446545920 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed260796776978432 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 152609420 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated96002572 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed254601015177246 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 trainer/global_step14449 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated248081434908672 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 248602320 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated41421329 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated261725524695040 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated260796881454592 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0gG<&#M I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated41421329 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated261725524695040 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated260796881454592 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed96002570 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated98317313 _step1154 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 152599750 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 % _timestamp1.7858017162496266e+09 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated248081434908672 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed248665869211648 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 248611992 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated12715446545920 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed56895982 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12684326106690 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated96002572 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed96002570 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated260796881454592 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated248665894115328 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 248611992 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed98317076 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated56895984 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed261725373318656 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed12684240303454 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed12715359109120 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed248081417869312 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed41421094 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated13059630579712 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed241916774873792 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13059504107008 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 152609420 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed248081417869312 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated254601118019842 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 248602320 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12715359109120 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12715446545920 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed260796776978432 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 152609420 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated96002572 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed254601015177246 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 trainer/global_step14449 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated248081434908672 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 248602320 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 )ت2#- _wandbruntime6429  _runtime6429?0 # trainer/global_step14449 /loss/FRt2/assignment_loss2.002043842568117 2loss/FBt2/assignment_loss9.68574589421678e-08 .loss/FRt2/detection_loss2.472682166274856 0loss/SRqqt1/detection_loss1.143426836654544 &loss/total_loss15.425920905666654 1loss/SRqqt1/assignment_loss0.781821801410988 1loss/SRqqt2/detection_loss1.3278711657971143 _step1155 0loss/FRt1/assignment_loss2.3350457598085272 3loss/FBt1/assignment_loss4.978720051654176e-07 /loss/FBt1/detection_loss1.0179498742389328 epoch0 % _timestamp1.7858017162513127e+09 _runtime6429.074141732 2loss/SRqqt2/assignment_loss0.8854588219369559 /loss/FRt1/detection_loss2.5400893211568873 /loss/FBt2/detection_loss0.9195307210902683N# /loss/FBt1/detection_loss1.0179498742389328 epoch0 % _timestamp1.7858017162513127e+09 _runtime6429.074141732 2loss/SRqqt2/assignment_loss0.8854588219369559 /loss/FRt1/detection_loss2.5400893211568873 /loss/FBt2/detection_loss0.9195307210902683 trainer/global_step14449 /loss/FRt2/assignment_loss2.002043842568117 2loss/FBt2/assignment_loss9.68574589421678e-08 .loss/FRt2/detection_loss2.472682166274856 0loss/SRqqt1/detection_loss1.143426836654544 &loss/total_loss15.425920905666654 1loss/SRqqt1/assignment_loss0.781821801410988 1loss/SRqqt2/detection_loss1.3278711657971143 _step1155 0loss/FRt1/assignment_loss2.3350457598085272 3loss/FBt1/assignment_loss4.978720051654176e-07 ǽe#(:  V disk.drbd2078.out0 disk.drbd1817.in0 disk.rbd1.in0" proc.memory.rssMB 11260.328125 disk.drbd2111.out0 disk.dm-0.in 4.64453125* proc.memory.percent17.181897163391113 network.sent 1.3301809e+07 disk.drbd1584.out0$ disk.nvme0n1.out281.99169921875 disk.drbd2041.in0 disk.drbd1552.in0 disk.rbd0.in0 disk.drbd2078.in0 disk.drbd1965.in0 network.recv741063 disk.drbd1584.in0 disk.sda2.in1.59375 disk.drbd1673.in0 disk.drbd2041.out0$ memory_percent41.25680327415466 disk.drbd1817.out0 disk.drbd1828.out0 disk.drbd1550.in0 disk.drbd1938.in0 disk.drbd1510.in0 disk.nvme0n1.in 4.64453125 disk.drbd1552.out0 disk.rbd0.out0 disk.sda.out 174.765625 cpu13.733937405618072 disk.drbd1356.out0 disk.drbd2118.out0 disk.sda.in1.59375 disk.sda1.in0 disk.drbd2111.in0 disk.drbd1931.out0 disk.drbd1226.in0* proc.memory.availableMB38497.94140625 proc.cpu.threads29 disk.drbd1510.out0 disk.sda2.out 174.765625 disk.drbd1938.out0 disk.drbd2118.in0 disk.drbd1673.out0 disk.drbd1289.out0 disk.rbd1.out 25.9150390625 disk.drbd1828.in0! disk.dm-0.out281.99169921875 disk.drbd1226.out0$ disk./.usageGB578.3506546020508 disk.drbd1289.in0 disk.sda1.out0 disk.drbd1285.in0 disk.drbd1285.out0( disk./.usagePercent32.3443962429875 disk.drbd1356.in0 disk.drbd1965.out0 disk.drbd1931.in0 disk.drbd1550.out0%&?#(: U gpu.0.gpu41.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts129.87' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent43.29 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0rbܚ#(: U gpu.0.gpu0.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts127.988' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.66266666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors06"h$(:  U disk.drbd1817.out0 disk.drbd1965.in0 disk.drbd1550.out0 disk.drbd1938.out0 disk.sda.in1.59375* proc.memory.availableMB38497.94140625 network.recv742771 disk.dm-0.in 4.64453125 disk.drbd1226.in0 disk.drbd1356.out0 disk.drbd2111.in0 disk.sda1.out0 disk.drbd1584.in0 disk.nvme0n1.in 4.64453125 disk.drbd1828.out0 disk.drbd1285.out0 disk.rbd1.out 25.9150390625 disk.drbd1289.out0 cpu13.734588919985361 network.sent 1.3346836e+07 disk.drbd1510.out0 disk.rbd1.in0 disk.drbd1965.out0 disk.drbd1931.out0* proc.memory.percent17.181897163391113 disk.drbd1510.in0$ disk./.usageGB578.3507118225098 disk.drbd1817.in0) disk./.usagePercent32.34439944305511 disk.drbd2118.out0 disk.drbd2041.in0" proc.memory.rssMB 11260.328125 proc.cpu.threads29 disk.drbd1552.out0 disk.drbd1828.in0 disk.drbd1550.in0 disk.drbd1285.in0 disk.drbd1584.out0$ disk.nvme0n1.out282.19091796875 disk.drbd1673.out0$ memory_percent41.25680327415466 disk.drbd2111.out0 disk.drbd1552.in0 disk.drbd1289.in0 disk.rbd0.in0 disk.drbd2078.out0 disk.drbd2118.in0 disk.drbd1931.in0 disk.drbd2041.out0 disk.drbd2078.in0 disk.sda1.in0! disk.dm-0.out282.19091796875 disk.rbd0.out0 disk.sda.out 175.0234375 disk.drbd1226.out0 disk.sda2.in1.59375 disk.sda2.out 175.0234375 disk.drbd1673.in0 disk.drbd1938.in0 disk.drbd1356.in02$- _wandbruntime6451  _runtime6451$ & lr-AdamW/pg10.0003736709318504531 & lr-AdamW/pg20.0003736709318504531 trainer/global_step14499 $ _timestamp1.785801738340353e+09 _runtime6451.163309085 _step1156aA$ _step1156 & lr-AdamW/pg10.0003736709318504531 & lr-AdamW/pg20.0003736709318504531 trainer/global_step14499 $ _timestamp1.785801738340353e+09 _runtime6451.163309085 %N2$- _wandbruntime6451  _runtime6451th$O ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed242737043317472 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed96328077 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current248 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated255464397453696 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 153126885 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed12758475624960 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 153126885 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed96328077 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed41560748 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed255464294611212 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 153117229 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated249509042838528 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12727251293740 trainer/global_step14499 _runtime6451.164587525 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated57088897 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed98649643 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 249454964 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed261681064881664 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 153117229 % _timestamp1.7858017383413007e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed248922589256704 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13103917891584 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated248922606296064 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 KEDeviceStatsMonitor.on_train_batchH}" _start/allocation.large_pool.current2 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12727337096864 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed248922589256704 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12758475624960 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated96328079 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12758563054592 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed261681064881664 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated261681169350656 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed262612803055104 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed57088895 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated41560996 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1>'$O VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated261681169350656 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated12758563054592 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated249509042838528 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12727251293740 trainer/global_step14499 _runtime6451.164587525 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated57088897 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed98649643 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 249454964 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed261681064881664 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 153117229 % _timestamp1.7858017383413007e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed248922589256704 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13103917891584 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated248922606296064 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12727337096864 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed248922589256704 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12758475624960 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated96328079 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12758563054592 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed261681064881664 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated261681169350656 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed262612803055104 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed57088895 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated41560996 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed242737043317472 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed96328077 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current248 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated255464397453696 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 153126885 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed12758475624960 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 153126885 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed96328077 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed41560748 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed255464294611212 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 153117229 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 JDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12759355244544 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed98656444 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12728214570198 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 249472144 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated98656691 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current247 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12759442681344 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated249526249955328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed248939756019712 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 153137422 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed261699111264256 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed96334720 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13104685595648 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed255481912256754 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 153137422 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 153127752 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed262630910647296 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated57092834 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 trainer/global_step14499 % _timestamp1.7858017387093127e+09 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated262631072509440 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated248939773059072 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current245 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0J&$M EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 249472144 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated98656691 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current247 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12759442681344 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated249526249955328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed248939756019712 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 153137422 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed261699111264256 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed96334720 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13104685595648 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed255481912256754 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 153137422 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 153127752 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed262630910647296 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated57092834 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 trainer/global_step14499 % _timestamp1.7858017387093127e+09 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated262631072509440 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated248939773059072 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current245 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated96334722 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed248939756019712 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed249526225051648 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12759355244544 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed98656444 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12728214570198 z2$- _wandbruntime6451  _runtime6451'$ /loss/FRt1/assignment_loss2.539571944247471 2loss/SRqqt1/assignment_loss0.8353613829163865 /loss/FRt2/detection_loss2.3853849169317183 trainer/global_step14499 _runtime6451.534172751 /loss/FRt2/assignment_loss2.572899783050621 .loss/FRt1/detection_loss2.418934319796187 0loss/SRqqt1/detection_loss1.280761244508331 1loss/SRqqt2/detection_loss1.2763179476364113 2loss/SRqqt2/assignment_loss0.9824950184435848 3loss/FBt1/assignment_loss5.043468794733249e-08 4loss/FBt2/assignment_loss5.7549288675514176e-08 &loss/total_loss16.227570437983257 epoch0 /loss/FBt1/detection_loss1.0605331234490643 /loss/FBt2/detection_loss0.8753106490195054 $ _timestamp1.785801738710966e+09 _step1159} $ 2loss/SRqqt2/assignment_loss0.9824950184435848 3loss/FBt1/assignment_loss5.043468794733249e-08 4loss/FBt2/assignment_loss5.7549288675514176e-08 &loss/total_loss16.227570437983257 epoch0 /loss/FBt1/detection_loss1.0605331234490643 /loss/FBt2/detection_loss0.8753106490195054 $ _timestamp1.785801738710966e+09 _step1159 /loss/FRt1/assignment_loss2.539571944247471 2loss/SRqqt1/assignment_loss0.8353613829163865 /loss/FRt2/detection_loss2.3853849169317183 trainer/global_step14499 _runtime6451.534172751 /loss/FRt2/assignment_loss2.572899783050621 .loss/FRt1/detection_loss2.418934319796187 0loss/SRqqt1/detection_loss1.280761244508331 1loss/SRqqt2/detection_loss1.2763179476364113 Ċ $(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts98.764' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent32.92133333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0_i$(:  V disk.sda2.in1.59375 disk.drbd1550.in0 disk.drbd1938.in0 disk.rbd0.out0 disk.dm-0.in 4.64453125 disk.drbd2078.out0 network.recv743729$ disk.nvme0n1.out282.54443359375 disk.drbd1828.in0 disk.drbd1938.out0* proc.memory.percent17.181897163391113 disk.drbd2111.out0 disk.drbd1550.out0 disk.sda.out 175.44921875 disk.drbd2078.in0 disk.drbd1584.in0 disk.drbd2041.in0 cpu13.735427274146973 disk.drbd1817.in0 disk.rbd1.in0 disk.rbd1.out 25.947265625" proc.memory.rssMB 11260.328125 proc.cpu.threads29$ disk./.usageGB578.3507308959961 disk.drbd1356.out0 disk.drbd1285.in0 disk.drbd1584.out0 disk.drbd1356.in0 disk.drbd2111.in0 disk.drbd1965.out0 disk.drbd1828.out0 disk.sda1.out0 disk.drbd2118.out0 disk.drbd1289.out0 disk.rbd0.in0 disk.sda2.out 175.44921875 disk.drbd1817.out0 disk.drbd1510.out0 disk.drbd1226.in0 network.sent 1.3349868e+07 disk.drbd1552.out0 disk.drbd1289.in0 disk.sda.in1.59375 disk.drbd1285.out0$ memory_percent41.25552773475647* proc.memory.availableMB38498.77734375 disk.nvme0n1.in 4.64453125 disk.drbd1965.in0 disk.drbd1931.in0 disk.drbd1510.in0! disk.dm-0.out282.54443359375 disk.drbd1552.in0 disk.drbd1226.out0) disk./.usagePercent32.34440050974431 disk.drbd1673.out0 disk.drbd2041.out0 disk.sda1.in0 disk.drbd2118.in0 disk.drbd1673.in0 disk.drbd1931.out0]W2$- _wandbruntime6473  _runtime6473 $ _runtime6473.820720447 _step1160 ' lr-AdamW/pg10.00037495954117471844 ' lr-AdamW/pg20.00037495954117471844 trainer/global_step14549 % _timestamp1.7858017609963539e+091$ _runtime6473.820720447 _step1160 ' lr-AdamW/pg10.00037495954117471844 ' lr-AdamW/pg20.00037495954117471844 trainer/global_step14549 % _timestamp1.7858017609963539e+09 ̮2$- _wandbruntime6473  _runtime6473˜'$O \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 % _timestamp1.7858017609970436e+09 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed249780927407104 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated256345294533204 _step1161 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated262583503636480 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated41703606 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 _runtime 6473.8209524 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 153654887 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12802471760384 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current250 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated96660229 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed262583399167488 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 153645231 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated262583503636480 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed263518332435968 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12771139757248 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed249780927407104 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated249780944446464 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 250315116 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current252 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated57285747 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated250369398678528 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13148958661120 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed96660227 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated12802559190016 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated98989353 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed262583399167488 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed96660227 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed57285745 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 _runtime 6473.8209524 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 153654887 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12802471760384 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current250 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated96660229 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed262583399167488 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 153645231 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated262583503636480 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 $>2$- _wandbruntime6474  _runtime6474p֨) $M J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated12803438816768 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 250322624 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 163959296 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated96666872 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12772103033706 trainer/global_step14549 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12803351379968 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated13149996258816 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed250386580891648 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed256362809336262 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 % _timestamp1.7858017613760011e+09 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 _runtime 6474.19935285 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated41706403 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated262601550026240 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated98996087 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed12803351379968 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated256362912178858 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated249798111209472 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated249798111209472 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed41706159 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated96666872 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed96666870 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated57289684 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed12772017230470 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated262601550026240 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed98995841 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current246 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed249798094170112 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated263536602054144 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed263536438094848 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed96666870 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed57289682 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 _step1162 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 153655754 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12803438816768 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed262601445550080 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated250386605795328 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13149857203200 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated243590809145152 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 250332296 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 250322624 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current244 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed249798094170112 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 250332296 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed98995841 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current246 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed249798094170112 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated263536602054144 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed263536438094848 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed96666870 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed57289682 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 _step1162 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 153655754 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12803438816768 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed262601445550080 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated250386605795328 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13149857203200 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated243590809145152 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 250332296 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 250322624 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current244 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed249798094170112 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 250332296 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated96666872 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12772103033706 trainer/global_step14549 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12803351379968 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated13149996258816 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed250386580891648 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed256362809336262 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 % _timestamp1.7858017613760011e+09 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 _runtime 6474.19935285 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated41706403 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated262601550026240 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated98996087 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed12803351379968 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated256362912178858 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated249798111209472 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated249798111209472 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed41706159 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated96666872 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ʨ2$- _wandbruntime6474  _runtime6474W $ 3loss/FBt2/assignment_loss4.110664911908755e-09 _runtime6474.200875808 0loss/SRqqt2/detection_loss1.087257442657926 trainer/global_step14549 4loss/FBt1/assignment_loss2.1581770355775612e-05 /loss/FRt1/detection_loss2.6000629284165124 /loss/FBt2/detection_loss0.8721197566402883 epoch0 $ _timestamp1.785801761378047e+09 /loss/FRt2/assignment_loss2.001350884820208 2loss/SRqqt1/assignment_loss0.8478437663654663 2loss/SRqqt2/assignment_loss0.8946267657297393 .loss/FRt2/detection_loss2.207296986638764 1loss/SRqqt1/detection_loss1.1055616116882967 /loss/FBt1/detection_loss0.6158404504259427 &loss/total_loss15.064954068225378 _step1163 /loss/FRt1/assignment_loss2.832971888961214A7$ .loss/FRt2/detection_loss2.207296986638764 1loss/SRqqt1/detection_loss1.1055616116882967 /loss/FBt1/detection_loss0.6158404504259427 &loss/total_loss15.064954068225378 _step1163 /loss/FRt1/assignment_loss2.832971888961214 3loss/FBt2/assignment_loss4.110664911908755e-09 _runtime6474.200875808 0loss/SRqqt2/detection_loss1.087257442657926 trainer/global_step14549 4loss/FBt1/assignment_loss2.1581770355775612e-05 /loss/FRt1/detection_loss2.6000629284165124 /loss/FBt2/detection_loss0.8721197566402883 epoch0 $ _timestamp1.785801761378047e+09 /loss/FRt2/assignment_loss2.001350884820208 2loss/SRqqt1/assignment_loss0.8478437663654663 2loss/SRqqt2/assignment_loss0.8946267657297393 P{$(: ϞU gpu.0.gpu23.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts126.403' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.13433333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0;ce$(:  U disk.drbd1356.in0 disk.rbd1.out 26.041015625 disk.drbd1965.in0 disk.drbd1673.out0 disk.sda.out 176.0703125 disk.drbd1584.out0$ memory_percent41.25596284866333 disk.drbd1828.in0 disk.drbd1289.out0 disk.drbd1285.in0 disk.drbd1285.out0 disk.drbd2078.out0 disk.drbd2118.in0 disk.drbd2041.in0 disk.drbd1828.out0 disk.drbd1550.out0 disk.drbd1226.out0 disk.drbd2078.in0 network.recv745271 disk.drbd1938.in0 disk.sda2.in1.59375 disk.drbd1817.out0) proc.memory.availableMB 38498.4921875 disk.drbd2111.out0 disk.drbd1931.in0 disk.drbd1552.in0 disk.sda.in1.59375 disk.drbd1356.out0 disk.drbd1965.out0$ disk./.usageGB578.3507118225098 disk.sda1.in0 disk.drbd1226.in0 disk.rbd1.in0 proc.cpu.threads29 disk.rbd0.in0 disk.drbd1673.in0* proc.memory.percent17.181897163391113) disk./.usagePercent32.34439944305511 disk.drbd1938.out0 disk.rbd0.out0 disk.drbd2111.in0 disk.drbd1817.in0! disk.dm-0.out282.77880859375 network.sent 1.339498e+07 disk.drbd2041.out0 disk.drbd1289.in0 disk.sda1.out0 disk.drbd1584.in0" proc.memory.rssMB 11260.328125 disk.drbd1550.in0 disk.drbd1510.in0 disk.drbd1931.out0 disk.dm-0.in 4.64453125 disk.nvme0n1.in 4.64453125$ disk.nvme0n1.out282.77880859375 cpu13.735786644128218 disk.drbd1552.out0 disk.drbd1510.out0 disk.drbd2118.out0 disk.sda2.out 176.07031253$(: U gpu.0.gpu28.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts128.885' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent42.961666666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors01e$(:  V disk.drbd1817.out0 disk.drbd2041.in0 disk.drbd1226.in0 disk.drbd1965.out0 disk.drbd1673.out0 disk.drbd1285.out0 disk.drbd2118.in0 disk.drbd2041.out0 disk.drbd1550.out0 disk.drbd1289.out0 network.sent 1.3440066e+07 disk.drbd2111.out0 disk.rbd1.out 26.103515625 disk.drbd1828.in0 disk.drbd1289.in0 disk.drbd1931.out0 disk.rbd0.in0 disk.drbd1285.in0 cpu13.736141901029526 disk.drbd1356.in0 disk.sda2.in1.59375 disk.drbd2111.in0 disk.drbd1552.in0 disk.drbd1550.in0 proc.cpu.threads29$ memory_percent41.25595688819885 disk.sda1.in0 disk.drbd1510.out0" disk.nvme0n1.out 283.001953125 disk.rbd1.in0 disk.sda1.out0 disk.drbd1817.in0 disk.drbd1673.in0 disk.dm-0.in 4.64453125* proc.memory.availableMB38498.49609375* proc.memory.percent17.181897163391113 disk.sda2.out 176.43359375 disk.rbd0.out0 disk.drbd2078.out0 disk.drbd1584.in0 disk.drbd1965.in0 disk.drbd1938.in0 disk.drbd1226.out0 disk.drbd1931.in0 disk.drbd1828.out0 disk.drbd1510.in0 disk.dm-0.out 283.001953125) disk./.usagePercent32.34439752301454 disk.drbd1356.out0 disk.drbd1552.out0 disk.drbd1938.out0 disk.sda.in1.59375 disk.sda.out 176.43359375 network.recv746986 disk.drbd2078.in0 disk.drbd1584.out0" proc.memory.rssMB 11260.328125$ disk./.usageGB578.3506774902344 disk.drbd2118.out0 disk.nvme0n1.in 4.64453125=N2$- _wandbruntime6496  _runtime6496$ ' lr-AdamW/pg10.00037624815049898375 ' lr-AdamW/pg20.00037624815049898375 trainer/global_step14599 % _timestamp1.7858017832184038e+09 _runtime6496.041368342 _step1164$ % _timestamp1.7858017832184038e+09 _runtime6496.041368342 _step1164 ' lr-AdamW/pg10.00037624815049898375 ' lr-AdamW/pg20.00037624815049898375 trainer/global_step14599 2$- _wandbruntime6496  _runtime6496>o'$O GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 _runtime6496.044876518 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed263485733453312 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated96992379 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated57482597 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated251229754518528 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 154182889 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated41845911 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13194108706816 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed251229729614848 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed263485733453312 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed41845633 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12815028220756 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated263485837922304 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed96992377 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated257226191612712 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12846467895808 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12815114023880 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated244411077588832 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed244411060549472 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 251165610 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated250639282596864 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 251175268 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 _step1165 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 154182889 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated99328508 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12846555325440 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed250639265557504 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13194251963904 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed257226088770228 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated264424006482432 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed96992377 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed250639265557504 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 143257088 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated263485837922304 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 154173233 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated250639282596864 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 trainer/global_step14599 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed99328228 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current278 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current280 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 168160768 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548tgv$O VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated263485837922304 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed96992377 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated257226191612712 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12846467895808 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12815114023880 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated244411077588832 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed244411060549472 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 251165610 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated250639282596864 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 251175268 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 _step1165 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 154182889 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated99328508 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12846555325440 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed250639265557504 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13194251963904 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed257226088770228 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated264424006482432 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed96992377 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed250639265557504 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 143257088 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 HBDeviceStatsMonit 'or.on_train_batch_start/oversize_allocations.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated263485837922304 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 154173233 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated250639282596864 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 trainer/global_step14599 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed99328228 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current278 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current280 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 168160768 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 _runtime6496.044876518 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed263485733453312 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated96992379 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated57482597 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated251229754518528 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 154182889 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated41845911 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13194108706816 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed251229729614848 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed263485733453312 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed41845633 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12815028220756 |j2$- _wandbruntime6496  _runtime64964y&$M O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated250656449359872 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated57486534 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated13195145182208 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated244427817761152 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed96999020 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 154183756 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed41848458 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated257243809258366 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 251182776 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated99335232 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current240 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 154193426 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12815991497214 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated96999022 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated250656449359872 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current242 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed244427800721792 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 154183756 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12847434952192 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated41848698 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _runtime6496.408181708 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed250656432320512 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed264441949149696 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed251246936731648 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed99334990 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 % _timestamp1.7858017835848427e+09 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed250656432320512 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 251192448 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 251192448 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed57486532 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 251182776 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13195012418048 _step1166 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated263503884312064 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0P &$M [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated250656449359872 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated57486534 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated13195145182208 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated244427817761152 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed96999020 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 154183756 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed41848458 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated257243809258366 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 251182776 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated99335232 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current240 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 154193426 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12815991497214 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated96999022 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated250656449359872 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current242 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed244427800721792 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 154183756 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12847434952192 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated41848698 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _runtime6496.408181708 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed250656432320512 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed264441949149696 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed251246936731648 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed99334990 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 % _timestamp1.7858017835848427e+09 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed250656432320512 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 251192448 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 251192448 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed57486532 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 251182776 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13195012418048 _step1166 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated263503884312064 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 2$- _wandbruntime6496  _runtime6496H $ 4loss/FBt1/assignment_loss1.6527213245887612e-05 4loss/FBt2/assignment_loss2.5558781378739088e-05 /loss/FBt2/detection_loss1.1484920397763037 % _timestamp1.7858017835864708e+09 2loss/SRqqt2/assignment_loss0.8897400452269484 /loss/FRt1/detection_loss2.6939888294690695 epoch0 1loss/SRqqt2/detection_loss1.0768176681564554 $loss/total_loss15.4157999362099 _runtime 6496.40928829 _step1167 .loss/FRt2/detection_loss2.360987770650536 1loss/SRqqt1/detection_loss0.9648666663893631 /loss/FBt1/detection_loss0.7381610693005116 trainer/global_step14599 /loss/FRt1/assignment_loss2.409992901178507 /loss/FRt2/assignment_loss2.299822773039341 2loss/SRqqt1/assignment_loss0.8328880870282384!%$ 4loss/FBt1/assignment_loss1.6527213245887612e-05 4loss/FBt2/assignment_loss2.5558781378739088e-05 /loss/FBt2/detection_loss1.1484920397763037 % _timestamp1.7858017835864708e+09 2loss/SRqqt2/assignment_loss0.8897400452269484 /loss/FRt1/detection_loss2.6939888294690695 epoch0 1loss/SRqqt2/detection_loss1.0768176681564554 $loss/total_loss15.4157999362099 _runtime 6496.40928829 _step1167 .loss/FRt2/detection_loss2.360987770650536 1loss/SRqqt1/detection_loss0.9648666663893631 /loss/FBt1/detection_loss0.7381610693005116 trainer/global_step14599 /loss/FRt1/assignment_loss2.409992901178507 /loss/FRt2/assignment_loss2.299822773039341 2loss/SRqqt1/assignment_loss0.8328880870282384 Nq"$(: ֒U gpu.0.gpu0.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts100.642' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent33.547333333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0˅§c$(:  U disk.drbd1552.in0 disk.drbd1817.in0 network.sent 1.3443098e+07 disk.drbd1510.out0 disk.drbd1356.out0 disk.drbd1673.out0 disk.drbd1285.in0 disk.rbd1.out 26.103515625# disk./.usageGB578.350715637207 disk.drbd1931.in0 disk.drbd1550.in0* proc.memory.availableMB38498.71484375 disk.drbd2118.in0 disk.drbd1965.in0 disk.sda.in1.59375 disk.sda1.in0 disk.drbd1510.in0 disk.drbd2078.out0 disk.drbd1584.in0 disk.drbd1965.out0 disk.drbd1289.out0 disk.drbd1552.out0 disk.drbd1938.out0 disk.rbd0.out0 disk.drbd2078.in0 disk.nvme0n1.in 4.64453125* proc.memory.percent17.181897163391113 disk.sda1.out0 disk.sda2.in1.59375 disk.rbd0.in0 disk.sda.out 176.73046875 disk.drbd1285.out0" disk.nvme0n1.out 283.283203125 disk.sda2.out 176.73046875 disk.drbd2111.out0 disk.dm-0.out 283.283203125 disk.drbd1817.out0 network.recv747813 disk.drbd2041.out0 disk.drbd1289.in0 disk.drbd1938.in0 disk.drbd1226.in0$ memory_percent41.25562310218811 disk.drbd2111.in0 disk.rbd1.in0 disk.drbd1673.in0 disk.drbd2041.in0 disk.drbd1828.in0 disk.drbd1226.out0 disk.drbd1584.out0 cpu13.73651680902459 disk.drbd2118.out0 disk.drbd1828.out0) disk./.usagePercent32.34439965639295 disk.drbd1931.out0 disk.drbd1550.out0 disk.dm-0.in 4.64453125 disk.drbd1356.in0" proc.memory.rssMB 11260.328125 proc.cpu.threads29k2$- _wandbruntime6518  _runtime6518wq$ % lr-AdamW/pg10.000377536759823249 % lr-AdamW/pg20.000377536759823249 trainer/global_step14649 % _timestamp1.7858018059133956e+09 _runtime6518.736378992 _step1168vp$ % lr-AdamW/pg20.000377536759823249 trainer/global_step14649 % _timestamp1.7858018059133956e+09 _runtime6518.736378992 _step1168 % lr-AdamW/pg10.000377536759823249 tL2$- _wandbruntime6518  _runtime6518ku, $O UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 _runtime6518.737726265 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed57679445 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed264388067739136 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated97324529 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12859002487388 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated264388172208128 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed97324527 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 _step1169 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed12890464031232 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13239432569856 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 154710891 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated12890551460864 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 154701235 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12858916684264 QDDevicjeeStatsMonitor.on_train_batch_start/inactive_split.all.allocated99668004 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 252025762 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated251497620747264 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed99667753 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed41988308 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed97324527 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current249 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated265329542928384 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed258106985849736 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated264388172208128 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated41988557 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated97324529 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12890551460864 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 252035420 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated251497620747264 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed251497603707904 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated252090110358528 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed245248069165472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 $ _timestamp1.785801805914403e+09 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12890464031232 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed251497603707904 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 trainer/global_step14649 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed252090085454848 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13239293507072 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated258107088692220 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated245248086204832 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed265329378961920 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated97324529 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12890551460864 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 252035420 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated251497620747264 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed251497603707904 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated252090110358528 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed245248069165472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 $ _timestamp1.785801805914403e+09 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12890464031232 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed251497603707904 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 trainer/global_step14649 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed252090085454848 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13239293507072 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated258107088692220 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated245248086204832 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed265329378961920 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed264388067739136 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated97324529 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12859002487388 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated264388172208128 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed97324527 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 _step1169 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed12890464031232 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13239432569856 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 154710891 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated12890551460864 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 154701235 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12858916684264 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated99668004 и2$- _wandbruntime6519  _runtime6519xF&$M VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed12891343650816 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed99674537 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 252052600 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated264406218597888 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 252052600 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed252107292571648 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated99674777 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated41991393 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated265347647597056 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed245264809337792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed97331170 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12891343650816 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _step1170 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated264406218597888 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed97331170 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed258124603495278 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated97331172 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 154711758 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 252042928 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated251514787510272 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 252042928 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed41991155 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated251514787510272 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated245264826377152 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12859879960722 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130+$M NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated97331172 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated57683384 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed265347494123520 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 trainer/global_step14649 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed251514770470912 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 252052600 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed252107292571648 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated99674777 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated41991393 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated265347647597056 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed245264809337792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current240 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed97331170 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12891343650816 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _step1170 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated264406218597888 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed97331170 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 S?DeviceStatsMonitor.on_train_batch_en{vUd/requested_bytes.all.freed258124603495278 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated97331172 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 154711758 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 252042928 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated251514787510272 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 252042928 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed41991155 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated251514787510272 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated245264826377152 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12859879960722 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed12891343650816 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed99674537 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 252052600 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated264406218597888 PDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated265290506493952 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated252355958897664 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed246085077781472 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12902805147772 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 _runtime6541.099384522 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated42132938 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current243 % _timestamp1.7858018282761095e+09 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated265290506493952 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 252885914 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed252355941858304 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed252950441294848 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed265290402024960 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current245 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed258987882929244 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed42132695 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed97656677 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 155238893 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13284620254208 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 155229237 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed97656677 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed252355941858304 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed12934460166656 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13284487482880 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated265290506493952 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated252355958897664 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed246085077781472 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12902805147772 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 _runtime6541.099384522 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated42132938 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current243 % _timestamp1.7858018282761095e+09 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated265290506493952 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 252885914 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed252355941858304 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed252950441294848 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed265290402024960 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current245 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed258987882929244 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed42132695 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed97656677 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 155238893 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13284620254208 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 155229237 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed97656677 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed252355941858304 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed12934460166656 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13284487482880 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 <2$- _wandbruntime6541  _runtime6541vCA$M JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated57880234 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed265308448407552 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 _step1174 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _runtime6541.467589651 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed42135505 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current235 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 155239760 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 252912752 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12935339786240 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated252373125660672 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12935427223040 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 100015972 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated265308552883712 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated266253202885120 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated97663322 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 155239760 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 252912752 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated259005603417382 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 $ _timestamp1.785801828644238e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed97663320 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current233 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed266253045217280 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed12903682620994 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 155249430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 252903080 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated13285529569792 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated246101834993152 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed252373108621312 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated97663322 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed97663320 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated12935427223040 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated252967673315328 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated265308552883712 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 100015737 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 252903080 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed252967648411648 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed252373108621312 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392\~&$M H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated252373125660672 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12935427223040 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 100015972 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated265308552883712 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated266253202885120 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated97663322 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 155239760 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 252912752 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated259005603417382 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 $ _timestamp1.785801828644238e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed97663320 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current233 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed266253045217280 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed12903682620994 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 155249430 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 252903080 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated13285529569792 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated246101834993152 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed252373108621312 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated97663322 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed97663320 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated12935427223040 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated252967673315328 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated265308552883712 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 100015737 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 252903080 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed252967648411648 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed252373108621312 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated57880234 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed265308448407552 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 _step1174 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _runtime6541.467589651 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed42135505 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current235 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 155239760 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 252912752 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12935339786240 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ]2$- _wandbruntime6541  _runtime6541p $ % _timestamp1.7858018286458828e+09 1loss/SRqqt1/detection_loss1.2704713923108935 epoch0 /loss/FRt1/detection_loss2.5132834289544896 0loss/SRqqt2/detection_loss1.156365106670329 -loss/FBt2/detection_loss1.27081481249922 0loss/FRt1/assignment_loss2.5559511015444625 3loss/FBt2/assignment_loss8.977916043059632e-05 /loss/FRt2/detection_loss2.3390942644113784 _runtime6541.468728513 _step1175 &loss/total_loss15.648525830385767 0loss/FRt2/assignment_loss2.0110041033262496 2loss/SRqqt1/assignment_loss1.0057752382167628 2loss/SRqqt2/assignment_loss0.6443139454690167 3loss/FBt1/assignment_loss2.520782021342427e-05 /loss/FBt1/detection_loss0.8813374500023201 trainer/global_step14699&&^$ -loss/FBt2/detection_loss1.27081481249922 0loss/FRt1/assignment_loss2.5559511015444625 3loss/FBt2/assignment_loss8.977916043059632e-05 /loss/FRt2/detection_loss2.3390942644113784 _runtime6541.468728513 _step1175 &loss/total_loss15.648525830385767 0loss/FRt2/assignment_loss2.0110041033262496 2loss/SRqqt1/assignment_loss1.0057752382167628 2loss/SRqqt2/assignment_loss0.6443139454690167 3loss/FBt1/assignment_loss2.520782021342427e-05 /loss/FBt1/detection_loss0.8813374500023201 trainer/global_step14699 % _timestamp1.7858018286458828e+09 1loss/SRqqt1/detection_loss1.2704713923108935 epoch0 /loss/FRt1/detection_loss2.5132834289544896 0loss/SRqqt2/detection_loss1.156365106670329 8qɛ$(: ƄU gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts125.846' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent41.94866666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0Gi$(:  ȚV disk.drbd1356.in0 disk.rbd1.out 26.2919921875" proc.memory.rssMB 11260.328125 disk.drbd1226.in0 disk.drbd1817.out0 disk.drbd1965.in0 disk.drbd1931.in0) disk./.usagePercent32.34441501671749 disk.drbd2118.in0 disk.drbd1673.out0 disk.drbd1552.out0 network.sent 1.3536193e+07 disk.sda.out 177.79296875 disk.drbd1552.in0 disk.drbd1289.in0 disk.dm-0.in 4.64453125 disk.drbd1356.out0 disk.drbd2111.out0$ memory_percent41.25627279281616* proc.memory.percent17.181897163391113 disk.drbd1550.in0 disk.sda1.in0$ disk.nvme0n1.out284.18701171875 disk.drbd1965.out0 disk.drbd2041.in0 disk.rbd0.in0 disk.drbd1226.out0 disk.sda2.out 177.79296875 disk.drbd1817.in0 disk.drbd2078.in0 disk.drbd1931.out0 disk.drbd2041.out0 network.recv751911 disk.drbd1550.out0 disk.drbd1510.in0 disk.drbd2078.out0 disk.nvme0n1.in 4.64453125 disk.sda2.in1.59375 disk.drbd2111.in0 disk.rbd1.in0 disk.drbd1828.in0 disk.drbd1828.out0 disk.drbd1289.out0 disk.drbd1938.in0 disk.sda1.out0) proc.memory.availableMB 38498.2890625 disk.drbd1285.in0 disk.drbd1584.out0 disk.drbd1673.in0 cpu13.736078609597048$ disk./.usageGB578.3509902954102 disk.drbd1938.out0 disk.drbd1285.out0 proc.cpu.threads29 disk.rbd0.out0 disk.sda.in1.59375 disk.drbd1510.out0! disk.dm-0.out284.18701171875 disk.drbd1584.in0 disk.drbd2118.out02$- _wandbruntime6563  _runtime6563B "$ trainer/global_step14749 % _timestamp1.7858018502479825e+09 _runtime6563.070951585 _step1176 & lr-AdamW/pg10.0003801139784717797 & lr-AdamW/pg20.0003801139784717797"ާ{$ % _timestamp1.7858018502479825e+09 _runtime6563.070951585 _step1176 & lr-AdamW/pg10.0003801139784717797 & lr-AdamW/pg20.0003801139784717797 trainer/global_step14749  BG2$- _wandbruntime6563  _runtime6563UD\ &$O UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13329698585600 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 253746066 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 155757239 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated253810822038528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed97988827 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated253214297048064 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current255 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated266192840779776 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 100348471 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed267140495720448 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated97988829 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated97988829 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed246922086397472 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated58073147 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed58073145 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated259868882851236 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 253755724 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 _runtime6563.072222448 _step1177 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated42275581 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed97988827 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 155757239 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12978456302080 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12946779414404 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12946693611280 # _timestamp1.78580185024896e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12978543731712 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13329829259776 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed266192736310784 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated267140651298304 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed259868780008752 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed253214280008704 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated12978543731712 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed12978456302080 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 155766895 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed266192736310784 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed253214280008704 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed253810797134848 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 100348471 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed267140495720448 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated97988829 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated97988829 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed246922086397472 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated58073147 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed58073145 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated259868882851236 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 253755724 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 _runtime6563.072222448 _step1177 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated42275581 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed97988827 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 155757239 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed12978456302080 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated12946779414404 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed12946693611280 # _timestamp1.78580185024896e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated12978543731712 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13329829259776 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed266192736310784 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated267140651298304 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed259868780008752 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed253214280008704 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated12978543731712 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed12978456302080 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 155766895 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed266192736310784 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed253214280008704 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed253810797134848 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 J8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated259886500496890 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated267158759452672 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed246938826569792 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated13330730297344 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 124375552 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13330605921792 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed97995470 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated97995472 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 155777432 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 100355239 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed42278157 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed267158610173440 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 155767762 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated253828029155328 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 trainer/global_step14749 % _timestamp1.7858018506188505e+09 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 155777432 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12979335921664 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 253763232 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12947656887738 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 155767762 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated266210887169536 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current257 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed12947571084502 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed266210782693376 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated12979423358464 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current255 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed97995470 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 253772904 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 100355496 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _step1178 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12979423358464 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed259886397654294 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251AĹ&$M L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 253772904 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 100355496 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _step1178 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated12979423358464 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed259886397654294 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated259886500496890 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated267158759452672 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed246938826569792 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated13330730297344 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 124375552 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13330605921792 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed97995470 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated97995472 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 155777432 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 100355239 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed42278157 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed267158610173440 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 155767762 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated253828029155328 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 trainer/global_step14749 % _timestamp1.7858018506188505e+09 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 155777432 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed12979335921664 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 253763232 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12947656887738 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 155767762 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated266210887169536 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current257 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed12947571084502 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed266210782693376 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated12979423358464 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current255 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed97995470 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ۻ2$- _wandbruntime6563  _runtime6563g$ 2loss/SRqqt1/assignment_loss0.7926598690156761 2loss/SRqqt2/assignment_loss0.8793567698448896 4loss/FBt2/assignment_loss4.3217549266690485e-06 1loss/SRqqt1/detection_loss1.0588397504889393 .loss/FRt2/detection_loss2.232014264558491 1loss/SRqqt2/detection_loss1.1583323407871258 epoch0 trainer/global_step14749 % _timestamp1.7858018506204927e+09 _runtime6563.443318985 _step1179 /loss/FBt1/detection_loss1.5878111213883932 0loss/FRt1/assignment_loss2.8097370717583634 /loss/FRt2/assignment_loss2.284863273720992 4loss/FBt1/assignment_loss1.3672509021740311e-05 /loss/FRt1/detection_loss2.5617729174654658 /loss/FBt2/detection_loss0.8920453440350218 &loss/total_loss16.257450717327306il$ /loss/FRt2/assignment_loss2.284863273720992 4loss/FBt1/assignment_loss1.3672509021740311e-05 /loss/FRt1/detection_loss2.5617729174654658 /loss/FBt2/detection_loss0.8920453440350218 &loss/total_loss16.257450717327306 2loss/SRqqt1/assignment_loss0.7926598690156761 2loss/SRqqt2/assignment_loss0.8793567698448896 4loss/FBt2/assignment_loss4.3217549266690485e-06 1loss/SRqqt1/detection_loss1.0588397504889393 .loss/FRt2/detection_loss2.232014264558491 1loss/SRqqt2/detection_loss1.1583323407871258 epoch0 trainer/global_step14749 % _timestamp1.7858018506204927e+09 _runtime6563.443318985 _step1179 /loss/FBt1/detection_loss1.5878111213883932 0loss/FRt1/assignment_loss2.8097370717583634 &$(: ͤU gpu.0.gpu32.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts125.655' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent41.885 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0ae$(:  Y\U) disk./.usagePercent32.34441501671749 disk.sda1.out0) proc.memory.availableMB 38498.0078125 disk.drbd1226.in0 disk.drbd2078.out0 network.sent 1.3581283e+07 network.recv753584 disk.drbd1550.in0 disk.dm-0.in 4.64453125" proc.memory.rssMB 11260.328125 disk.rbd0.out0 disk.drbd2078.in0 disk.drbd1356.in0 disk.sda2.in1.59375* proc.memory.percent17.181897163391113 disk.drbd1931.out0$ disk./.usageGB578.3509902954102 disk.drbd1673.out0 disk.rbd0.in0 disk.drbd1510.out0 disk.drbd1356.out0 disk.drbd1938.in0 disk.sda.in1.59375$ disk.nvme0n1.out284.44482421875 disk.drbd1817.in0 cpu13.735715203336811 disk.rbd1.in0 disk.drbd1965.in0 disk.drbd1931.in0 disk.drbd1584.out0% memory_percent41.256701946258545 proc.cpu.threads29 disk.drbd1965.out0 disk.drbd1285.in0 disk.drbd1289.out0 disk.drbd1938.out0 disk.drbd1510.in0 disk.drbd1584.in0 disk.sda2.out 178.01171875 disk.drbd2111.out0 disk.drbd1552.in0! disk.dm-0.out284.44482421875 disk.drbd2118.out0 disk.rbd1.out 26.3779296875 disk.drbd1550.out0 disk.sda1.in0 disk.drbd1226.out0 disk.drbd2041.in0 disk.drbd2041.out0 disk.drbd1828.out0 disk.nvme0n1.in 4.64453125 disk.drbd1817.out0 disk.drbd1552.out0 disk.drbd2118.in0 disk.drbd2111.in0 disk.drbd1828.in0 disk.drbd1673.in0 disk.drbd1289.in0 disk.sda.out 178.01171875 disk.drbd1285.out0\<$(: U gpu.0.gpu0.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts95.556' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.851999999999997 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0>L i$(:  U disk.drbd1938.out0! disk.dm-0.out285.72998046875 disk.sda2.out 178.27734375 cpu13.73535260645014 disk.drbd2041.in0 disk.drbd1552.out0 disk.sda.out 178.27734375 disk.drbd1226.in0 disk.drbd1289.out0 disk.drbd1510.in0 disk.drbd1584.in0 disk.drbd2118.in0 disk.drbd1550.in0 disk.drbd2118.out0) disk./.usagePercent32.34441928347431$ disk./.usageGB578.3510665893555 disk.drbd1965.out0 disk.drbd1550.out0 disk.sda1.out0 disk.drbd1285.in0 disk.drbd2078.out0$ memory_percent41.25675559043884 network.sent 1.3626268e+07 disk.rbd1.out 26.4404296875 disk.drbd1285.out0 disk.drbd1584.out0 disk.drbd1356.in0 disk.drbd1673.out0 disk.drbd1289.in0 disk.nvme0n1.in 4.64453125* proc.memory.percent17.181897163391113 disk.drbd1817.in0 disk.drbd1673.in0 disk.drbd1828.out0 disk.sda.in1.59375 disk.sda1.in0 disk.dm-0.in 4.64453125 disk.drbd1226.out0* proc.memory.availableMB38497.97265625 disk.drbd2041.out0 disk.drbd1356.out0 disk.drbd1965.in0 disk.drbd1938.in0" proc.memory.rssMB 11260.328125 proc.cpu.threads29 disk.drbd1817.out0 disk.rbd1.in0 disk.rbd0.out0 disk.drbd2111.in0 disk.drbd1828.in0 disk.drbd2078.in0 network.recv754992 disk.drbd2111.out0 disk.drbd1931.out0 disk.drbd1552.in0 disk.rbd0.in0$ disk.nvme0n1.out285.72998046875 disk.sda2.in1.59375 disk.drbd1931.in0 disk.drbd1510.out0#PZ 2$- _wandbruntime6585  _runtime6585_xaD$ & lr-AdamW/pg20.0003814025877960449 trainer/global_step14799 $ _timestamp1.785801872512664e+09 _runtime6585.335645128 _step1180 & lr-AdamW/pg10.0003814025877960449dƙ$ & lr-AdamW/pg10.0003814025877960449 & lr-AdamW/pg20.0003814025877960449 trainer/global_step14799 $ _timestamp1.785801872512664e+09 _runtime6585.335645128 _step1180 2$- _wandbruntime6585  _runtime6585:'$O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated267095175065600 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed98320977 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed42417421 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 % _timestamp1.7858018725136273e+09 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated58269997 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 _runtime6585.340654446 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated13022539867136 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13022452437504 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13375033833984 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13022539867136 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 254615876 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 trainer/global_step14799 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated267095175065600 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated98320979 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated42417659 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 _step1181 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 156285241 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 126479872 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 156294897 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated268046211712512 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed98320977 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed267095070596608 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed254072618159104 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed254671152974848 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated98320979 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated254072635198464 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed254072618159104 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13374907354112 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated247759112052832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 100687656 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated254072635198464 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed13022452437504 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current238 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated254671177878528 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed260749677088260'$O VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated267095175065600 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated98320979 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated42417659 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 _step1181 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 156285241 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 126479872 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 156294897 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated268046211712512 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed98320977 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed267095070596608 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed254072618159104 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed254671152974848 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated98320979 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated254072635198464 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed254072618159104 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13374907354112 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated247759112052832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 100687656 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated254072635198464 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed13022452437504 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current238 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated254671177878528 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed260749677088260 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated267095175065600 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed98320977 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed42417421 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 % _timestamp1.7858018725136273e+09 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated58269997 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 _runtime6585.340654446 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated13022539867136 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13022452437504 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13375033833984 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current240 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13022539867136 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 254615876 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 trainer/global_step14799 62$- _wandbruntime6585  _runtime6585=w`$M D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed13023332057088 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated267113221455360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed254089784922112 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed254688360091648 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 120181248 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated267113221455360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 254623384 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated98327622 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed58273932 _runtime 6585.70432888 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13023419493888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed254089784922112 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated98327622 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current243 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13023332057088 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed267113116979200 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 156295764 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated268064330094592 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed260767294733802 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 254633056 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13023419493888 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed247775835185792 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 100694381 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated254688384995328 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13375824918016 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12991545351246 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 % _timestamp1.7858018728810008e+09 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated254089801961472 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated254089801961472 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 145084928 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed268064185009664 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed98327620 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 254633056 REDeviceStatsMonitor.Gygon_train_batch_end/active_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 156305434 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed12991459548010 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 trainer/global_step14799 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed13023332057088 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated267113221455360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed254089784922112 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed254688360091648 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 120181248 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated267113221455360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 254623384 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated98327622 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed58273932 _runtime 6585.70432888 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13023419493888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed254089784922112 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated98327622 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current243 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13023332057088 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed267113116979200 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 156295764 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated268064330094592 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed260767294733802 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 254633056 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13023419493888 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed247775835185792 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 100694381 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated254688384995328 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13375824918016 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated12991545351246 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 % _timestamp1.7858018728810008e+09 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated254089801961472 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated254089801961472 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 145084928 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed268064185009664 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed98327620 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 254633056 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 156305434 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed12991459548010 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 trainer/global_step14799 f2$- _wandbruntime6585  _runtime6585mк $ 4loss/FBt1/assignment_loss1.0927514123200884e-07 /loss/FRt1/detection_loss2.4589655094030425 epoch0 _step1183 0loss/FRt2/assignment_loss1.9796200283368428 2loss/SRqqt1/assignment_loss0.7064112524888266 1loss/SRqqt2/assignment_loss0.829840224370173 /loss/FRt2/detection_loss2.4437672113378843 0loss/SRqqt2/detection_loss1.237625816538639 &loss/total_loss15.154451314040783 trainer/global_step14799 % _timestamp1.7858018728826406e+09 0loss/FRt1/assignment_loss2.1761750404427693 3loss/FBt2/assignment_loss3.369527333300463e-06 1loss/SRqqt1/detection_loss0.9393557192630727 .loss/FBt1/detection_loss1.321506964120393 /loss/FBt2/detection_loss1.0611800689366646 _runtime6585.705448853G}<$ 1loss/SRqqt1/detection_loss0.9393557192630727 .loss/FBt1/detection_loss1.321506964120393 /loss/FBt2/detection_loss1.0611800689366646 _runtime6585.705448853 4loss/FBt1/assignment_loss1.0927514123200884e-07 /loss/FRt1/detection_loss2.4589655094030425 epoch0 _step1183 0loss/FRt2/assignment_loss1.9796200283368428 2loss/SRqqt1/assignment_loss0.7064112524888266 1loss/SRqqt2/assignment_loss0.829840224370173 /loss/FRt2/detection_loss2.4437672113378843 0loss/SRqqt2/detection_loss1.237625816538639 &loss/total_loss15.154451314040783 trainer/global_step14799 % _timestamp1.7858018728826406e+09 0loss/FRt1/assignment_loss2.1761750404427693 3loss/FBt2/assignment_loss3.369527333300463e-06 _VV$(:  U disk.drbd1356.out0 disk.drbd1817.in0$ disk./.usageGB578.3508033752441 disk.drbd1584.in0 disk.drbd1931.out0 disk.drbd2041.in0 disk.drbd1552.in0 cpu13.734992349463452 disk.drbd1938.out0 disk.drbd1226.out0 disk.drbd1965.in0 disk.drbd1965.out0 disk.sda2.out 178.79296875 disk.drbd1226.in0& proc.memory.availableMB 38498.1875 disk.drbd1356.in0 disk.drbd2111.in0 disk.drbd1938.in0 disk.drbd1510.out0 disk.drbd1828.in0 disk.drbd1285.out0 disk.drbd1510.in0 disk.sda2.in1.59375 disk.nvme0n1.in 4.64453125 disk.nvme0n1.out286.25 disk.drbd1817.out0 disk.drbd1673.out0 disk.dm-0.out286.25 proc.cpu.threads29 disk.drbd1289.in0$ proc.memory.rssMB11260.33203125 disk.sda1.in0 disk.drbd1285.in0 network.recv756005 disk.drbd1584.out0 disk.drbd2118.in0 disk.drbd1673.in0 disk.rbd0.out0 disk.drbd2078.out0* disk./.usagePercent32.344404563163295 disk.drbd2111.out0 disk.drbd1931.in0 disk.drbd1550.in0 disk.drbd1289.out0 disk.dm-0.in 4.64453125 disk.drbd2078.in0 disk.drbd2118.out0 disk.drbd1552.out0 disk.drbd1828.out0 disk.drbd1550.out0 disk.rbd0.in0 disk.sda.out 178.79296875 network.sent 1.3629305e+07 disk.rbd1.in0 disk.drbd2041.out0 disk.sda1.out0) proc.memory.percent17.18190312385559 disk.rbd1.out 26.5341796875 disk.sda.in1.59375$ memory_percent41.25642776489258ZZ$(: U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts102.607' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent34.202333333333335 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0w2$- _wandbruntime6606  _runtime6606PI$ _runtime6606.937133189 _step1184 ' lr-AdamW/pg10.00038269119712031025 ' lr-AdamW/pg20.00038269119712031025 trainer/global_step14849 % _timestamp1.7858018941141076e+09?r $ % _timestamp1.7858018941141076e+09 _runtime6606.937133189 _step1184 ' lr-AdamW/pg10.00038269119712031025 ' lr-AdamW/pg20.00038269119712031025 trainer/global_step14849 42$- _wandbruntime6606  _runtime6606lw'$O @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed261630574167768 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed255531508814848 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed98653127 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated42560804 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current285 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 _runtime6606.941655518 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 156813243 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 156813243 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed254930956309504 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated268951770618368 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13420236899840 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13420108322816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated254930973348864 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 101027651 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13066448572928 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed13034470538296 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 trainer/global_step14849 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13066536002560 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated254930973348864 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated261630677010252 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated98653129 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 255476028 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated267997509351424 % _timestamp1.7858018941151128e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed254930956309504 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 156822899 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed248596103629472 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed58466845 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed267997404882432 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated255531533718528 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated13034556341420 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated248596120668832 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 156822899 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current283 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed268951617137664 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated98653129 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed13066448572928 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 101027366 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed42560521qu$O F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 255466370 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated58466847 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated267997509351424 % _timestamp1.7858018941151128e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed254930956309504 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 156822899 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed248596103629472 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed58466845 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed267997404882432 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated255531533718528 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated13034556341420 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated248596120668832 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 156822899 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current283 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed268951617137664 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated98653129 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed13066448572928 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 101027366 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed42560521 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed261630574167768 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed255531508814848 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed98653127 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated42560804 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current285 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 _runtime6606.941655518 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 156813243 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 156813243 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed254930956309504 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated268951770618368 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13420236899840 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13420108322816 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated254930973348864 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 101027651 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13066448572928 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed13034470538296 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 trainer/global_step14849 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13066536002560 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated254930973348864 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated261630677010252 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated98653129 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 255476028 JDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 101034198 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed58470782 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 255493208 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated268015555741184 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated42563674 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed13067328192512 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13421007798784 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated13035433814754 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated254948140111872 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed13035348011518 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated268969883495424 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated248612860841152 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed268015451265024 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 101034458 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 255483536 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current258 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 156833436 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13067415629312 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated255548740835328 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed261648191813310 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 255493208 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 156823766 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _runtime6607.307435656 _step1186 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated268015555741184 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current260&$M JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 % _timestamp1.7858018944840872e+09 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated98659772 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated13421142660096 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13067328192512 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 101034198 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed58470782 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 255493208 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated268015555741184 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated42563674 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed13067328192512 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13421007798784 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated13035433814754 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated254948140111872 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed13035348011518 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated268969883495424 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated248612860841152 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed268015451265024 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 101034458 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 255483536 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current258 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 156833436 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13067415629312 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated255548740835328 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed261648191813310 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 255493208 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 156823766 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _runtime6607.307435656 _step1186 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated268015555741184 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current260 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 156823766 B2$- _wandbruntime6607  _runtime6607> $ 2loss/SRqqt2/assignment_loss0.8698527443167918 % _timestamp1.7858018944859917e+09 /loss/FRt1/assignment_loss2.550426030700857 2loss/SRqqt1/assignment_loss0.8170060926764992 4loss/FBt1/assignment_loss2.1308480873116764e-06 .loss/FRt2/detection_loss2.608772478169865 &loss/total_loss15.619917211844053 epoch0 trainer/global_step14849 4loss/FBt2/assignment_loss1.0361939070199137e-06 1loss/SRqqt1/detection_loss1.1673647805582732 /loss/FBt1/detection_loss1.1915773572400212 /loss/FBt2/detection_loss0.6955760192269316 _step1187 .loss/FRt1/detection_loss2.425843291153962 1loss/SRqqt2/detection_loss0.9269768327646771 _runtime6607.308817973 /loss/FRt2/assignment_loss2.366518417994181DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed268899739168256 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 trainer/global_step14899 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13110444708352 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13110532137984 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 157350901 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated262511574089760 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 101367889 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 256326522 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 % _timestamp1.7858019165818143e+09 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed98985277 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed255789294459904 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated268899843637248 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed255789294459904 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed98985277 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated98985279 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 _runtime6629.405044762 _step1189 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated269857281757696 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated249433129284832 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated13110532137984 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated256391889558528 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 157350901 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated268899843637248 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current246 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13465253136384 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed58663695 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed262511471247276 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed13078359001804'$O ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated255789311499264 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 101367641 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed268899739168256 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 256336180 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed269857117791232 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13465392199168 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated268899843637248 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed255789294459904 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed98985277 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated98985279 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 _runtime6629.405044762 _step1189 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated269857281757696 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated249433129284832 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated13110532137984 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated256391889558528 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 157350901 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated268899843637248 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current246 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13465253136384 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed58663695 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed262511471247276 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed13078359001804 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated98985279 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed268899739168256 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 trainer/global_step14899 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13110444708352 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13110532137984 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 157350901 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated262511574089760 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 101367889 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 256326522 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 % _timestamp1.7858019165818143e+09 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed98985277 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed255789294459904 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 $ _timestamp1.785801916959324e+09 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated262529191735414 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 157361438 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed268917785550848 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13111324327936 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated13079322278262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed58667632 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed13079236475026 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed42706783 trainer/global_step14899 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated98991922 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed98991920 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 101374415 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed255806461222912 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 157361438 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated255806478262272 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13111411764736 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 256353360 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed249449852417792 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated268917890027008 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 157351768 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated13466287653888 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13466156986880 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 256353360 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated255806478262272 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current247 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated256409096675328 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 _step1190 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13111411764736]Z$M ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 $ _timestamp1.785801916959324e+09 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated262529191735414 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 157361438 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed268917785550848 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13111324327936 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated13079322278262 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed58667632 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed13079236475026 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed42706783 trainer/global_step14899 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated98991922 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed98991920 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 101374415 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed255806461222912 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 157361438 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated255806478262272 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13111411764736 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 256353360 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed249449852417792 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak,o1L  2712961024 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated268917890027008 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 157351768 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated13466287653888 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13466156986880 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 256353360 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated255806478262272 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current247 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated256409096675328 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 _step1190 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13111411764736 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed269875228758528 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _runtime6629.782723077 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 101374662 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed13111324327936 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated98991922 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated42707028 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 !2%- _wandbruntime6629  _runtime66295 % 3loss/FBt1/assignment_loss3.916868089553256e-07 2loss/FBt2/assignment_loss0.003040079583943994 1loss/SRqqt1/detection_loss1.0314088951498788 /loss/FRt2/assignment_loss2.146584478360188 .loss/FRt1/detection_loss2.365383963379049 &loss/total_loss14.445210079299976 trainer/global_step14899 /loss/FRt2/detection_loss2.4773339897016937 /loss/FBt1/detection_loss0.7905812742454665 epoch0 % _timestamp1.7858019169613886e+09 _step1191 0loss/FRt1/assignment_loss2.0147699922630466 2loss/SRqqt1/assignment_loss0.5752289791901907 2loss/SRqqt2/assignment_loss0.8301810404719335 1loss/SRqqt2/detection_loss1.0457533663781038 /loss/FBt2/detection_loss1.1649436288896728 _runtime6629.784218104Ղ% /loss/FRt2/assignment_loss2.146584478360188 .loss/FRt1/detection_loss2.365383963379049 &loss/total_loss14.445210079299976 trainer/global_step14899 /loss/FRt2/detection_loss2.4773339897016937 /loss/FBt1/detection_loss0.7905812742454665 epoch0 % _timestamp1.7858019169613886e+09 _step1191 0loss/FRt1/assignment_loss2.0147699922630466 2loss/SRqqt1/assignment_loss0.5752289791901907 2loss/SRqqt2/assignment_loss0.8301810404719335 1loss/SRqqt2/detection_loss1.0457533663781038 /loss/FBt2/detection_loss1.1649436288896728 _runtime6629.784218104 3loss/FBt1/assignment_loss3.916868089553256e-07 2loss/FBt2/assignment_loss0.003040079583943994 1loss/SRqqt1/detection_loss1.0314088951498788 ܛ%(: ֙U gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts130.892' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent43.63066666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0^Z=l%(:  ӇV disk.drbd1226.in0 disk.drbd2111.in0 disk.drbd1817.in0 disk.rbd1.in0 disk.drbd1552.out0 disk.drbd1938.in0 disk.rbd0.out0 disk.drbd1828.out0 disk.drbd2041.out0 disk.drbd1552.in0) disk./.usagePercent32.34440264312273 disk.drbd1289.out0 disk.drbd1584.out0 disk.drbd2118.out0 disk.drbd1356.out0) proc.memory.percent17.18190312385559 disk.sda.out 179.83984375 disk.sda1.out0 disk.drbd1226.out0 disk.drbd1673.in0% memory_percent41.257280111312866 disk.drbd1938.out0 disk.drbd2111.out0 disk.drbd1965.in0 disk.drbd1673.out0 cpu13.734181035618072 disk.drbd1550.in0 disk.rbd0.in0 disk.drbd1510.in0! disk.dm-0.out287.46533203125 disk.drbd1356.in0* proc.memory.availableMB38497.62890625 disk.drbd1931.in0 disk.drbd1285.out0 disk.dm-0.in 4.64453125 disk.drbd2078.out0 disk.sda2.out 179.83984375 disk.drbd1965.out0 disk.drbd2041.in0$ proc.memory.rssMB11260.33203125$ disk./.usageGB578.3507690429688 disk.drbd1510.out0 disk.sda.in1.59375 disk.drbd2078.in0 disk.drbd2118.in0 disk.rbd1.out 26.5966796875 disk.sda2.in1.59375 proc.cpu.threads29 network.sent 1.3719777e+07 disk.drbd1584.in0 disk.nvme0n1.in 4.64453125 disk.drbd1817.out0 network.recv759113 disk.drbd1550.out0 disk.drbd1285.in0 disk.drbd1931.out0 disk.drbd1289.in0$ disk.nvme0n1.out287.46533203125 disk.drbd1828.in0 disk.sda1.in0F؜%(: U gpu.0.gpu2.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts95.653' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.884333333333338 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0{f%(:  U disk.drbd2078.out0 cpu13.733769808123416 disk.drbd1552.in0 disk.drbd1828.in0 disk.rbd0.out0 disk.drbd1226.out0 disk.rbd1.out 26.62890625 disk.drbd2041.out0 disk.sda1.in0 disk.drbd1285.out0) proc.memory.availableMB 38498.3359375 disk.drbd1552.out0 disk.drbd1584.out0$ disk.nvme0n1.out287.79638671875 disk.sda2.out 180.3203125 disk.drbd1965.in0 disk.drbd2111.out0! disk.dm-0.out287.79638671875 disk.dm-0.in 4.64453125 disk.drbd1356.out0 disk.rbd1.in0 disk.drbd1965.out0 disk.drbd1550.in0 disk.drbd1289.out0 disk.rbd0.in0 disk.drbd1356.in0$ disk./.usageGB578.3507690429688 disk.drbd1828.out0 disk.drbd2118.out0 disk.drbd1817.out0 disk.sda.out 180.3203125 disk.drbd1510.out0 disk.drbd1584.in0 disk.drbd2118.in0 disk.drbd1289.in0 disk.sda.in1.59375 network.recv760786 disk.drbd1673.out0 disk.drbd2041.in0 disk.drbd2111.in0 disk.drbd1931.out0 disk.nvme0n1.in 4.64453125 disk.sda2.in1.59375$ proc.memory.rssMB11260.33203125 network.sent 1.3764841e+07 disk.drbd1673.in0 disk.drbd1938.in0) disk./.usagePercent32.34440264312273 disk.drbd1938.out0 disk.sda1.out0 disk.drbd1510.in0 disk.drbd1226.in0) proc.memory.percent17.18190312385559 proc.cpu.threads29 disk.drbd2078.in0$ memory_percent41.25620126724243 disk.drbd1817.in0 disk.drbd1931.in0 disk.drbd1550.out0 disk.drbd1285.in0c 2%- _wandbruntime6651  _runtime6651f% & lr-AdamW/pg10.0003852684157688408 & lr-AdamW/pg20.0003852684157688408 trainer/global_step14949 % _timestamp1.7858019385638573e+09 _runtime6651.386839996 _step1192) C% & lr-AdamW/pg10.0003852684157688408 & lr-AdamW/pg20.0003852684157688408 trainer/global_step14949 % _timestamp1.7858019385638573e+09 _runtime6651.386839996 _step1192 j2%- _wandbruntime6651  _runtime66510'%O 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated270762766789120 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed13154440843776 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated42847355 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 _step1193 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated13154528273408 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13154440843776 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current264 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed13122247465312 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated99317429 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 257196332 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 157878903 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 257186674 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13154528273408 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed263392368326784 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated99317429 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed257252220494848 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 101707902 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13510521390592 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed256647632610304 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed42847093 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13510386522112 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed250270120861472 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed269802073454080 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 157878903 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated269802177923072 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed99317427 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 % _timestamp1.7858019385648243e+09 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated250270137900832 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed270762607016960 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current262 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed257252220494848 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 101707902 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13510521390592 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed256647632610304 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed42847093 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13510386522112 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed250270120861472 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed269802073454080 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 157878903 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated269802177923072 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed99317427 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 % _timestamp1.7858019385648243e+09 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated250270137900832 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed270762607016960 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current262 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated13154528273408 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13154440843776 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current264 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed13122247465312 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated99317429 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 257196332 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 157878903 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 257186674 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13154528273408 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed263392368326784 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated99317429 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated269820224312832 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13511291240960 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 157889440 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed99324070 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed256664799373312 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13155407900160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current264 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated42850106 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed257269427611648 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated13511421907968 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed250286861033792 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 157879770 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13155320463360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated99324072 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 101714590 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 257203840 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed263409985972326 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 257203840 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13155407900160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed99324070 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed42849844 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed13123124938534 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 101714326 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current262 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated13123210741770 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 257213512 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated99324072 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated269820224312832 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 157879770 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 157889440 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 _runtime6651.760794479 _step1194 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 257213512 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed13155320463360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated256664816412672 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated263410088814922 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated269820224312832 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13511291240960 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 157889440 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed99324070 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed256664799373312 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13155407900160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current264 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated42850106 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed257269427611648 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated13511421907968 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed250286861033792 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 157879770 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13155320463360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated99324072 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 101714590 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 257203840 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed263409985972326 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 257203840 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13155407900160 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed99324070 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed42849844 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed13123124938534 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 101714326 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current262 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated13123210741770 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 257213512 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated99324072 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated269820224312832 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 157879770 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 157889440 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 _runtime6651.760794479 _step1194 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 257213512 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed13155320463360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated256664816412672 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated263410088814922 82%- _wandbruntime6651  _runtime6651[ % 0loss/FRt1/assignment_loss2.4993666010824116 3loss/FBt2/assignment_loss6.409433347552242e-06 trainer/global_step14949 % _timestamp1.7858019389390883e+09 _step1195 /loss/FBt2/detection_loss1.0676151798285831 epoch0 3loss/FBt1/assignment_loss4.207385854094943e-08 /loss/FRt1/detection_loss2.3999439762397246 /loss/FRt2/detection_loss2.3177535580027673 /loss/FBt1/detection_loss0.6557755737072405 &loss/total_loss15.015077804475785 _runtime 6651.76191825 0loss/FRt2/assignment_loss2.1728732039531073 2loss/SRqqt1/assignment_loss0.7652893434270469 0loss/SRqqt2/assignment_loss0.91593425593047 0loss/SRqqt1/detection_loss1.079817032808779 1loss/SRqqt2/detection_loss1.1407026279884487lj7% 0loss/FRt1/assignment_loss2.4993666010824116 3loss/FBt2/assignment_loss6.409433347552242e-06 trainer/global_step14949 % _timestamp1.7858019389390883e+09 _step1195 /loss/FBt2/detection_loss1.0676151798285831 epoch0 3loss/FBt1/assignment_loss4.207385854094943e-08 /loss/FRt1/detection_loss2.3999439762397246 /loss/FRt2/detection_loss2.3177535580027673 /loss/FBt1/detection_loss0.6557755737072405 &loss/total_loss15.015077804475785 _runtime 6651.76191825 0loss/FRt2/assignment_loss2.1728732039531073 2loss/SRqqt1/assignment_loss0.7652893434270469 0loss/SRqqt2/assignment_loss0.91593425593047 0loss/SRqqt1/detection_loss1.079817032808779 1loss/SRqqt2/detection_loss1.1407026279884487 emՐ%(: ﮏU gpu.0.gpu23.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts126.297' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent42.099 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0i%(:  U$ disk./.usageGB578.3507499694824 disk.drbd1226.out0 disk.drbd1965.in0 disk.drbd1673.in0 disk.drbd1356.out0 proc.cpu.threads29 disk.drbd1584.out0% memory_percent41.256242990493774 disk.rbd1.in0 disk.rbd1.out 26.72265625 disk.drbd1552.out0 disk.sda1.in0! disk.dm-0.out288.05029296875 cpu13.733416876160039 network.recv761802 disk.drbd1828.out0 disk.drbd1289.in0 disk.drbd1510.in0 disk.dm-0.in 4.64453125 disk.drbd2118.out0 disk.drbd1938.out0) proc.memory.percent17.18190312385559 network.sent 1.3767916e+07 disk.drbd1552.in0 disk.nvme0n1.in 4.64453125 disk.drbd1356.in0 disk.drbd1817.in0* disk./.usagePercent32.344401576433526 disk.drbd1550.out0 disk.rbd0.out0 disk.drbd2118.in0$ disk.nvme0n1.out288.05029296875 disk.drbd2041.out0 disk.rbd0.in0 disk.drbd1285.out0 disk.drbd2078.in0* proc.memory.availableMB38498.30859375$ proc.memory.rssMB11260.33203125 disk.drbd1828.in0 disk.drbd1938.in0 disk.sda.out 180.6796875 disk.drbd1584.in0 disk.drbd1931.in0 disk.drbd1931.out0 disk.drbd1285.in0 disk.drbd1510.out0 disk.drbd1226.in0 disk.drbd2078.out0 disk.sda2.in1.59375 disk.drbd2111.in0 disk.drbd1965.out0 disk.drbd1673.out0 disk.drbd1550.in0 disk.drbd2041.in0 disk.drbd1289.out0 disk.sda1.out0 disk.drbd2111.out0 disk.drbd1817.out0 disk.sda.in1.59375 disk.sda2.out 180.6796875&2%- _wandbruntime6673  _runtime6673wU[% ' lr-AdamW/pg10.00038655702509310613 ' lr-AdamW/pg20.00038655702509310613 trainer/global_step14999 % _timestamp1.7858019607418337e+09 _runtime6673.564804644 _step1196 /% _step1196 ' lr-AdamW/pg10.00038655702509310613 ' lr-AdamW/pg20.00038655702509310613 trainer/global_step14999 % _timestamp1.7858019607418337e+09 _runtime6673.564804644 2%- _wandbruntime6673  _runtime66735A'%O ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated264273368248776 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 158397249 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 258046826 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13198436979200 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated258112601238528 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated257505987800064 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 258056484 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current263 _step1197 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed258112576334848 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 trainer/global_step14999 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated270704512208896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated13166221731944 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13555539674624 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed13166135928820 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed99649577 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated99649579 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 102047422 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated13198524408832 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed257505970760704 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed99649577 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed42989764 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed270704407739904 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed270704407739904 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed271668116009472 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed13198436979200 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated59057397 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated270704512208896 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated257505987800064 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated271668277878784 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13198524408832 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated99649579 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current261 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated257505987800064 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 258056484 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current263 _step1197 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed258112576334848 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 trainer/global_step14999 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated270704512208896 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated13166221731944 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13555539674624 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed13166135928820 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed99649577 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated99649579 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 102047422 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated13198524408832 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed257505970760704 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed99649577 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed42989764 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed270704407739904 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed270704407739904 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed271668116009472 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed13198436979200 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated59057397 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated270704512208896 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated257505987800064 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated271668277878784 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 t2%- _wandbruntime6673  _runtime6673|&%M WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated264290985894430 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 158417442 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated13556575080960 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated99656222 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 102053915 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated13167099205278 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated257523154563072 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current254 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 258063992 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 258063992 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 158407772 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated59061334 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated271686383436288 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed271686227865600 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed264290883051834 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed59061332 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed257523137523712 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed99656220 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated42992835 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed251123869649792 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated99656222 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 258073664 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 258073664 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13199404035584 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed258129783451648 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated99656222 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 102053915 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated13167099205278 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated257523154563072 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current254 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 258063992 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 258063992 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 158407772 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated59061334 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated271686383436288 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed271686227865600 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed264290883051834 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed59061332 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed257523137523712 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed99656220 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated42992835 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed251123869649792 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated99656222 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 258073664 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 258073664 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13199404035584 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 LP2%- _wandbruntime6673  _runtime6673< % epoch0 trainer/global_step14999 2loss/FBt1/assignment_loss4.11142447297329e-05 3loss/FBt2/assignment_loss0.0008835940440221624 &loss/total_loss16.358690654921045 _runtime6673.934513636 _step1199 0loss/FRt1/assignment_loss2.5175718470495574 /loss/FRt2/assignment_loss2.196955731002296 2loss/SRqqt1/assignment_loss1.0918375724212315 2loss/SRqqt2/assignment_loss0.9160387418462872 /loss/FBt1/detection_loss1.0933063023171767 % _timestamp1.7858019611113565e+09 /loss/FRt1/detection_loss2.3183064638656012 /loss/FRt2/detection_loss2.4361509978089755 1loss/SRqqt1/detection_loss1.3607943380504088 1loss/SRqqt2/detection_loss1.1984415152823673 .loss/FBt2/detection_loss1.228362436988391i5% _runtime6673.934513636 _step1199 0loss/FRt1/assignment_loss2.5175718470495574 /loss/FRt2/assignment_loss2.196955731002296 2loss/SRqqt1/assignment_loss1.0918375724212315 2loss/SRqqt2/assignment_loss0.9160387418462872 /loss/FBt1/detection_loss1.0933063023171767 % _timestamp1.7858019611113565e+09 /loss/FRt1/detection_loss2.3183064638656012 /loss/FRt2/detection_loss2.4361509978089755 1loss/SRqqt1/detection_loss1.3607943380504088 1loss/SRqqt2/detection_loss1.1984415152823673 .loss/FBt2/detection_loss1.228362436988391 epoch0 trainer/global_step14999 2loss/FBt1/assignment_loss4.11142447297329e-05 3loss/FBt2/assignment_loss0.0008835940440221624 &loss/total_loss16.358690654921045 "$q%(: U gpu.0.gpu22.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts126.437' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.14566666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors03 f%(:  U disk.drbd1356.in0 disk.drbd1356.out0* proc.memory.availableMB38497.88671875* disk./.usagePercent32.344401576433526$ disk./.usageGB578.3507499694824 disk.drbd1828.out0 disk.sda1.out0 disk.drbd1584.out0 disk.drbd1285.in0 disk.dm-0.out 288.380859375 disk.drbd2118.out0$ memory_percent41.25688672065735 disk.drbd1828.in0 disk.sda.in1.59375 disk.drbd1226.in0 disk.sda2.out 180.98828125 disk.sda1.in0 disk.drbd2118.in0 disk.drbd1938.out0 disk.rbd0.in0 disk.drbd2078.out0 disk.drbd2111.in0 disk.drbd1817.out0 disk.rbd1.out 26.78515625 disk.drbd1285.out0 proc.cpu.threads29 network.recv763418 disk.dm-0.in 4.64453125 disk.sda2.in1.59375 disk.drbd1673.out0$ proc.memory.rssMB11260.33203125 disk.drbd1550.out0 disk.drbd1289.out0 disk.drbd1817.in0 disk.drbd1673.in0 disk.drbd2041.out0 cpu13.732915632406515 disk.drbd1289.in0 network.sent 1.3812957e+07 disk.rbd0.out0 disk.drbd2078.in0 disk.drbd1584.in0 disk.rbd1.in0) proc.memory.percent17.18190312385559 disk.drbd2111.out0 disk.drbd1931.in0 disk.drbd1226.out0 disk.drbd1552.in0 disk.drbd1550.in0 disk.drbd1510.in0 disk.drbd1552.out0 disk.drbd1510.out0 disk.drbd1965.out0 disk.drbd2041.in0 disk.drbd1938.in0 disk.sda.out 180.98828125 disk.nvme0n1.in 4.64453125" disk.nvme0n1.out 288.380859375 disk.drbd1965.in0 disk.drbd1931.out0WF5%(: ԷU gpu.0.gpu24.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts127.633ʹ6' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.54533333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0S`%(:  U% proc.memory.availableMB 38497.875 cpu13.732546042786916 network.sent 1.385795e+07 disk.dm-0.out 288.603515625 disk.drbd1226.in0 disk.drbd2118.out0 disk.drbd1673.out0 disk.drbd1289.out0 disk.drbd1938.out0 disk.sda1.in0 disk.sda2.in1.59375 disk.drbd1552.in0 disk.drbd2118.in0 disk.nvme0n1.in 4.64453125 disk.drbd1673.in0 disk.drbd1931.out0 disk.drbd1510.out0 disk.drbd1965.in0$ disk./.usageGB578.3507499694824 disk.drbd1828.in0$ memory_percent41.25690460205078 disk.drbd2111.out0$ proc.memory.rssMB11260.33203125 disk.drbd1289.in0 disk.rbd0.in0 disk.rbd0.out0 disk.drbd2078.out0 disk.rbd1.out 26.78515625 disk.drbd2078.in0 disk.drbd1817.in0 disk.drbd2041.out0 disk.drbd1285.out0 disk.drbd1356.out0 disk.drbd1552.out0 disk.sda.in1.59375 disk.dm-0.in 4.64453125 disk.drbd1226.out0 disk.sda2.out 181.28515625 disk.drbd1965.out0 disk.drbd1510.in0 disk.drbd1938.in0 disk.drbd1285.in0 disk.drbd1584.in0 disk.drbd1817.out0 network.recv764949* disk./.usagePercent32.344401576433526 disk.drbd1584.out0 disk.drbd1828.out0 disk.sda1.out0 disk.drbd2111.in0) proc.memory.percent17.18190312385559 proc.cpu.threads29 disk.drbd1356.in0 disk.rbd1.in0 disk.drbd1931.in0 disk.drbd1550.in0 disk.sda.out 181.28515625" disk.nvme0n1.out 288.603515625 disk.drbd2041.in0 disk.drbd1550.out0N2%- _wandbruntime6696  _runtime6696% trainer/global_step15049 $ _timestamp1.785801983238367e+09 _runtime6696.061337005 _step1200 ' lr-AdamW/pg10.00038784563441737144 ' lr-AdamW/pg20.00038784563441737144oq&% _runtime6696.061337005 _step1200 ' lr-AdamW/pg10.00038784563441737144 ' lr-AdamW/pg20.00038784563441737144 trainer/global_step15049 $ _timestamp1.785801983238367e+09 o2%- _wandbruntime6696  _runtime6696rE'%O =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated43132990 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated258364325950464 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 102387237 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed271606742025728 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13600845124608 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed271606742025728 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed59254245 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated271606846494720 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed99981727 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 trainer/global_step15049 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 158934907 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated13242520544256 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed258364308911104 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed258364308911104 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 158925251 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13600708158976L4'%O YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed271606742025728 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13600845124608 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed271606742025728 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed59254245 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated271606846494720 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed99981727 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 trainer/global_step15049 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 158934907 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated13242520544256 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed258364308911104 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed258364308911104 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 158925251 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13600708158976 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated43132990 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated258364325950464 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 102387237 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 i2%- _wandbruntime6696  _runtime6696&%M ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed251960878265792 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 158935774 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed13243312734208 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 158945444 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated258990164195328 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed59258182 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed43135521 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 258933816 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed272591748401152 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed13210901865550 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13243312734208 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13601609109504 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated265171882973938 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed258381475674112 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed99988370 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 258933816 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 258924144 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed271624788408320 trainer/global_step15049 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 102393703 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 158935774 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed13243312734208 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 158945444 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated258990164195328 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed59258182 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed43135521 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 258933816 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed272591748401152 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed13210901865550 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13243312734208 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13601609109504 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated265171882973938 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed258381475674112 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed99988370 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 258933816 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 258924144 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed271624788408320 trainer/global_step15049 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 102393703 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 $42%- _wandbruntime6696  _runtime6696#% 1loss/SRqqt1/detection_loss1.1301930231257127 epoch0 /loss/FRt1/assignment_loss2.339084525193487 2loss/SRqqt2/assignment_loss0.7709758581236358 loss/FBt1/assignment_loss0 3loss/FBt2/assignment_loss5.279249634274461e-07 0loss/SRqqt2/detection_loss1.206257914196989 /loss/FBt2/detection_loss0.7387800207527887 &loss/total_loss14.713647702656093 trainer/global_step15049 /loss/FRt1/detection_loss2.5244391046109653 /loss/FBt1/detection_loss0.6237913260443343 % _timestamp1.7858019836088254e+09 /loss/FRt2/assignment_loss2.299171284020665 2loss/SRqqt1/assignment_loss0.7508178603727944 /loss/FRt2/detection_loss2.3301362582897567 _runtime6696.431753977 _step1203ߨ% 2loss/SRqqt2/assignment_loss0.7709758581236358 loss/FBt1/assignment_loss0 3loss/FBt2/assignment_loss5.279249634274461e-07 0loss/SRqqt2/detection_loss1.206257914196989 /loss/FBt2/detection_loss0.7387800207527887 &loss/total_loss14.713647702656093 trainer/global_step15049 /loss/FRt1/detection_loss2.5244391046109653 /loss/FBt1/detection_loss0.6237913260443343 % _timestamp1.7858019836088254e+09 /loss/FRt2/assignment_loss2.299171284020665 2loss/SRqqt1/assignment_loss0.7508178603727944 /loss/FRt2/detection_loss2.3301362582897567 _runtime6696.431753977 _step1203 1loss/SRqqt1/detection_loss1.1301930231257127 epoch0 /loss/FRt1/assignment_loss2.339084525193487 ,d%(: U gpu.0.gpu41.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts100.978' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent33.65933333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0Tc%(:  V disk.drbd1817.in0 cpu13.732307035867265 disk.drbd1356.out0 disk.drbd1289.out0 disk.dm-0.out 288.869140625$ proc.memory.rssMB11260.33203125 disk.drbd1938.in0 disk.drbd1226.out0 network.recv765995" disk.nvme0n1.out 288.869140625 disk.drbd1289.in0 disk.drbd1938.out0$ memory_percent41.25707149505615 network.sent 1.3861018e+07$ disk./.usageGB578.3507690429688 disk.drbd1673.in0 disk.drbd1673.out0 disk.drbd1285.out0 disk.drbd1356.in0 disk.sda2.in1.59375 disk.drbd1931.in0 disk.drbd1552.in0 disk.drbd1552.out0 disk.drbd1828.out0 disk.drbd2078.in0 disk.drbd1828.in0 disk.drbd1550.out0 disk.rbd1.out 26.94140625 disk.dm-0.in 4.64453125( proc.memory.availableMB 38497.765625 disk.drbd1965.out0 disk.sda.out 181.90234375 disk.drbd1285.in0 proc.cpu.threads29 disk.drbd1965.in0 disk.sda1.out0 disk.drbd1510.in0) proc.memory.percent17.18190312385559 disk.sda1.in0 disk.drbd1584.out0 disk.drbd2118.in0) disk./.usagePercent32.34440264312273 disk.nvme0n1.in 4.64453125 disk.drbd1550.in0 disk.drbd1510.out0 disk.drbd1226.in0 disk.drbd2111.in0 disk.drbd1817.out0 disk.rbd1.in0 disk.drbd2041.in0 disk.sda.in1.59375 disk.drbd1584.in0 disk.drbd2118.out0 disk.rbd0.in0 disk.rbd0.out0 disk.sda2.out 181.90234375 disk.drbd2111.out0 disk.drbd1931.out0 disk.drbd2078.out0 disk.drbd2041.out02B2%- _wandbruntime6718  _runtime6718k=% % _timestamp1.7858020055012352e+09 _runtime 6718.32419389 _step1204 & lr-AdamW/pg10.0003891342437416368 & lr-AdamW/pg20.0003891342437416368 trainer/global_step15099q8% % _timestamp1.7858020055012352e+09 _runtime 6718.32419389 _step1204 & lr-AdamW/pg10.0003891342437416368 & lr-AdamW/pg20.0003891342437416368 trainer/global_step15099 2%- _wandbruntime6718  _runtime67182]'%O TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed43275237 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed259222647061504 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed59451095 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed272509076311552 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed273479127874048 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed259833288014848 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 100313879 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed13286429250048 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed259222647061504 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated252781163748832 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 259767130 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13645839859200 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed266035059565308 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated272509180780544 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current274 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated43275511 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887X %O TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed43275237 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed259222647061504 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed59451095 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed272509076311552 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed273479127874048 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed259833288014848 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 100313879 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed13286429250048 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed259222647061504 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated252781163748832 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 259767130 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13645839859200 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed266035059565308 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated272509180780544 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current274 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated43275511 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 Kw2%- _wandbruntime6718  _runtime6718CQ&%M OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current250 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed43278025 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed13254790329058 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed259239813824512 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed266052677210850 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated59455034 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed13287308869632 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 259784296 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 102733057 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 _step1206 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated43278275 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 259793968 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 159473446 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 259784296 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated13646878398976 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 102733309 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 159473446 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated252797903921152 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13287396306432 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated273497398434304 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13646747731968 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated272527227170304 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed252797886881792 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 trainer/global_step15099 % _timestamp1.7858020058818753e+09 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 159463776 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 259793968 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated259239830863872 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed59455032 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13287308869632 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936׀E&%M WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated259239830863872 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed59455032 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13287308869632 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current250 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed43278025 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed13254790329058 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed259239813824512 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed266052677210850 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated59455034 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed13287308869632 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 259784296 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 102733057 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 _step1206 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated43278275 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 259793968 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 159473446 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 259784296 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated13646878398976 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 102733309 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 159473446 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated252797903921152 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13287396306432 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated273497398434304 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13646747731968 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated272527227170304 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed252797886881792 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 trainer/global_step15099 % _timestamp1.7858020058818753e+09 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 159463776 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 259793968 =l2%- _wandbruntime6718  _runtime6718/ I# % 0loss/FRt2/assignment_loss1.9394459829610937 .loss/FRt2/detection_loss2.342428372186773 _runtime 6718.70633273 2loss/SRqqt1/assignment_loss0.7927515775164855 /loss/FBt1/detection_loss0.7057287113063715 /loss/FBt2/detection_loss1.5800171142274684 0loss/FRt1/assignment_loss2.4808534247965754 4loss/FBt1/assignment_loss1.6255810596600317e-08 3loss/FBt2/assignment_loss7.347234382590076e-06 .loss/FRt1/detection_loss2.526755438575262 1loss/SRqqt2/detection_loss1.0962415010716509 &loss/total_loss15.547736254736206 epoch0 _step1207 1loss/SRqqt2/assignment_loss0.878510639139708 0loss/SRqqt1/detection_loss1.204996129464624 trainer/global_step15099 % _timestamp1.7858020058834996e+09n% .loss/FRt2/detection_loss2.342428372186773 _runtime 6718.70633273 2loss/SRqqt1/assignment_loss0.7927515775164855 /loss/FBt1/detection_loss0.7057287113063715 /loss/FBt2/detection_loss1.5800171142274684 0loss/FRt1/assignment_loss2.4808534247965754 4loss/FBt1/assignment_loss1.6255810596600317e-08 3loss/FBt2/assignment_loss7.347234382590076e-06 .loss/FRt1/detection_loss2.526755438575262 1loss/SRqqt2/detection_loss1.0962415010716509 &loss/total_loss15.547736254736206 epoch0 _step1207 1loss/SRqqt2/assignment_loss0.878510639139708 0loss/SRqqt1/detection_loss1.204996129464624 trainer/global_step15099 % _timestamp1.7858020058834996e+09 0loss/FRt2/assignment_loss1.9394459829610937 )-_%(:  ܕU) proc.memory.percent17.18190312385559 disk.drbd1550.in0 disk.drbd1938.out0 disk.sda.in1.59375 disk.drbd1817.out0 disk.drbd1552.out0 disk.drbd1584.out0" disk.nvme0n1.out 289.083984375 disk.rbd1.out 27.03515625 disk.drbd1965.out0* disk./.usagePercent32.344401576433526 disk.drbd1828.in0 disk.drbd1289.in0 disk.rbd0.out0 disk.drbd1510.out0 disk.dm-0.in 4.64453125 disk.sda2.out 182.28125 proc.cpu.threads29 disk.sda1.out0 disk.drbd2118.out0) proc.memory.availableMB 38497.7578125 network.recv767414 disk.drbd1510.in0 disk.drbd2111.out0 disk.drbd1673.in0$ proc.memory.rssMB11260.33203125 disk.drbd1938.in0 disk.drbd2118.in0 disk.drbd1552.in0 disk.drbd1828.out0 disk.drbd2111.in0 disk.drbd1817.in0 disk.drbd1965.in0 disk.drbd2041.out0 cpu13.731757681311269 disk.drbd1584.in0 disk.drbd1285.in0 disk.drbd1931.out0 disk.drbd1289.out0 disk.dm-0.out 289.083984375 disk.nvme0n1.in 4.64453125$ memory_percent41.25708341598511 disk.rbd0.in0 disk.sda1.in0 disk.drbd1285.out0 disk.rbd1.in0 disk.drbd1673.out0 disk.sda.out 182.28125 disk.drbd1356.in0 disk.drbd2041.in0 disk.drbd2078.in0 disk.drbd1356.out0 disk.drbd1931.in0 network.sent 1.3905967e+07$ disk./.usageGB578.3507499694824 disk.drbd1550.out0 disk.drbd2078.out0 disk.drbd1226.out0 disk.drbd1226.in0 disk.sda2.in1.59375)%(: U gpu.0.gpu29.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts127.223' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent42.407666666666664 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0x쳛%(: U gpu.0.gpu35.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts130.04' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent43.346666666666664 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors08Oe%(:  U network.sent 1.3951129e+07 disk.drbd1550.in0! disk.dm-0.out289.30322265625 disk.drbd1584.out0 cpu13.731430717909854 disk.drbd1552.out0 disk.drbd2078.out0 disk.drbd1289.out0 disk.sda1.in0 disk.drbd1356.out0 disk.drbd1938.out0 disk.drbd1226.in0 disk.drbd1226.out0 disk.drbd1356.in0 disk.drbd1931.out0 disk.drbd1510.in0 disk.drbd1817.out0 disk.drbd1673.out0 disk.sda2.in1.59375 disk.sda2.out 182.5078125% memory_percent41.256749629974365 disk.drbd2078.in0' disk./.usagePercent32.344391549555 disk.nvme0n1.in 4.64453125 disk.dm-0.in 4.64453125 disk.drbd2118.out0 disk.drbd1289.in0 disk.rbd0.out0 disk.drbd1965.in0 disk.drbd2041.in0 disk.drbd1828.out0 disk.rbd0.in0 disk.sda.out 182.5078125 disk.sda1.out0 disk.drbd2111.out0 disk.drbd1552.in0 disk.rbd1.out 27.09765625) proc.memory.availableMB 38497.9765625 disk.drbd1510.out0 disk.drbd1817.in0 disk.rbd1.in0 disk.drbd1965.out0 network.recv768948 disk.drbd1938.in0 disk.drbd1584.in0 disk.drbd2118.in0$ disk.nvme0n1.out289.30322265625 disk.drbd1673.in0$ proc.memory.rssMB11260.33203125 proc.cpu.threads29 disk.sda.in1.59375 disk.drbd2111.in0 disk.drbd2041.out0$ disk./.usageGB578.3505706787109 disk.drbd1828.in0 disk.drbd1550.out0 disk.drbd1285.in0 disk.drbd1285.out0 disk.drbd1931.in0) proc.memory.percent17.18190312385559#v92%- _wandbruntime6740  _runtime6740 ]% ' lr-AdamW/pg10.00039042285306590207 ' lr-AdamW/pg20.00039042285306590207 trainer/global_step15149 $ _timestamp1.785802027707648e+09 _runtime6740.530633795 _step1208E% _step1208 ' lr-AdamW/pg10.00039042285306590207 ' lr-AdamW/pg20.00039042285306590207 trainer/global_step15149 $ _timestamp1.785802027707648e+09 _runtime6740.530633795 2%- _wandbruntime6740  _runtime6740Zx%O MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13691192767488 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 159981255 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed273411410597376 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 103065922 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 103066158 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed260080985211904 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 159981255 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated260081002251264 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed273411410597376 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 100646029 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 260627282 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed260080985211904 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated59647947 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated274384861526016 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated13297887122468 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed260693643854848 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 trainer/global_step15149 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated273411515066368 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current236 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed59647945 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated273411515066368 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 UGDeviceStatsMonit-3Cor.on_train_batch_start/allocation.small_pool.allocated 159990911 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current234 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed253618155325472 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated260693668758528 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 100646027 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed274384701753856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed273411410597376 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 103065922 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 103066158 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed260080985211904 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 159981255 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated260081002251264 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed273411410597376 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 100646029 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 260627282 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed260080985211904 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated59647947 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated274384861526016 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated13297887122468 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed260693643854848 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 trainer/global_step15149 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated273411515066368 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current236 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed59647945 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated273411515066368 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 159990911 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current234 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed253618155325472 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated260693668758528 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 100646027 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed274384701753856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed43420737 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated13692093727232 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed13331305005056 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed274402803546112 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated266933677132954 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 159991778 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed273429456979968 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 trainer/global_step15149 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 100652672 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13331305005056 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed59651882 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 141152768 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 100652672 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 100652670 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated273429561456128 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated43420997 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed260710850971648 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 260654120 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated273429561456128 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 160001448 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated274402969602560 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 166056448 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed253634895497792 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed260098151974912 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 260644448 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current260 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 260644448 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 160001448 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13331392441856 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 103072619 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated260710875875328%M K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 159991778 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed273429456979968 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 trainer/global_step15149 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 100652672 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13331305005056 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed59651882 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 141152768 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 100652672 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 100652670 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated273429561456128 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated43420997 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed260710850971648 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 260654120 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated273429561456128 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 160001448 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated274402969602560 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 166056448 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed253634895497792 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed260098151974912 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 260644448 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current260 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 L=DeviJ.yih ceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 260644448 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 160001448 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13331392441856 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 103072619 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated260710875875328 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13331392441856 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed43420737 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated13692093727232 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed13331305005056 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed274402803546112 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated266933677132954 ڴr2%- _wandbruntime6740  _runtime6740w'$q % 4loss/FBt2/assignment_loss1.8209975666222974e-06 /loss/FRt1/detection_loss2.3564644277571363 1loss/SRqqt1/detection_loss1.1486880244058255 /loss/FBt1/detection_loss0.7350038802609421 0loss/FRt1/assignment_loss2.4273193516629807 /loss/FRt2/assignment_loss2.071239146996628 2loss/SRqqt2/assignment_loss0.8970555814218977 3loss/FBt1/assignment_loss7.078872071220898e-06 &loss/total_loss14.664104890400964 epoch0 2loss/SRqqt1/assignment_loss0.6400381343548077 1loss/SRqqt2/detection_loss1.2051070403111608 trainer/global_step15149 % _timestamp1.7858020280820997e+09 _step1211 .loss/FRt2/detection_loss2.193106526085599 /loss/FBt2/detection_loss0.9900738772743478 _runtime6740.904925419ͱ\% 2loss/SRqqt1/assignment_loss0.6400381343548077 1loss/SRqqt2/detection_loss1.2051070403111608 trainer/global_step15149 % _timestamp1.7858020280820997e+09 _step1211 .loss/FRt2/detection_loss2.193106526085599 /loss/FBt2/detection_loss0.9900738772743478 _runtime6740.904925419 4loss/FBt2/assignment_loss1.8209975666222974e-06 /loss/FRt1/detection_loss2.3564644277571363 1loss/SRqqt1/detection_loss1.1486880244058255 /loss/FBt1/detection_loss0.7350038802609421 0loss/FRt1/assignment_loss2.4273193516629807 /loss/FRt2/assignment_loss2.071239146996628 2loss/SRqqt2/assignment_loss0.8970555814218977 3loss/FBt1/assignment_loss7.078872071220898e-06 &loss/total_loss14.664104890400964 epoch0 ㋛%(: T gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts128.408' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.80266666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0HBZa%(:  V disk.dm-0.in 4.64453125 disk.drbd2118.in0 disk.drbd2118.out0 disk.rbd0.out0% memory_percent41.256844997406006 network.sent 1.3954205e+07 disk.nvme0n1.in 4.64453125 disk.drbd1817.out0 disk.drbd1285.out0 proc.cpu.threads29 disk.drbd1673.in0 disk.drbd2041.out0 disk.drbd1828.out0 disk.drbd1550.in0 disk.drbd1510.in0 disk.drbd1510.out0 disk.drbd1584.in0$ disk.nvme0n1.out289.57275390625 disk.drbd2041.in0 disk.drbd1938.out0 disk.drbd1552.in0 disk.drbd1552.out0 disk.sda.out183.0625 disk.drbd1584.out0 disk.drbd1356.in0 disk.sda2.out183.0625 disk.drbd1817.in0 disk.drbd1931.in0 disk.drbd1289.in0 disk.drbd1289.out0 cpu13.731048183004988) proc.memory.availableMB 38497.9140625$ disk./.usageGB578.3511161804199 disk.drbd1226.in0 disk.drbd1356.out0 disk.drbd1828.in0! disk.dm-0.out289.57275390625 disk.drbd2111.out0 disk.sda1.in0$ proc.memory.rssMB11260.33203125 network.recv770105) disk./.usagePercent32.34442205686624 disk.drbd1965.out0 disk.rbd0.in0 disk.sda2.in1.59375 disk.drbd1965.in0 disk.sda1.out0 disk.drbd2111.in0 disk.rbd1.out 27.09765625 disk.drbd1285.in0 disk.drbd1226.out0 disk.drbd2078.in0 disk.drbd1550.out0 disk.sda.in1.59375 disk.rbd1.in0 disk.drbd1673.out0 disk.drbd1938.in0) proc.memory.percent17.18190312385559 disk.drbd2078.out0 disk.drbd1931.out0fuf2%- _wandbruntime6762  _runtime6762j% & lr-AdamW/pg10.0003917114623901674 & lr-AdamW/pg20.0003917114623901674 trainer/global_step15199 % _timestamp1.7858020497999675e+09 _runtime6762.622941117 _step1212% _step1212 & lr-AdamW/pg10.0003917114623901674 & lr-AdamW/pg20.0003917114623901674 trainer/global_step15199 % _timestamp1.7858020497999675e+09 _runtime6762.622941117 ޹2%- _wandbruntime6762  _runtime6762 U'%O QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current254 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 100978177 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13736390287360 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 100978177 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated59844797 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed43561031 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13736251224576 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed267796853724324 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated274313849352192 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed260939323362304 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed260939323362304 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13374421520896 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 _step1213 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 $ _timestamp1.785802049800905e+09 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 160518913 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed13341689782852 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 261497092 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 103405826 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 100978179 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated260939340401664 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13374508950528 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated260939340401664 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed275290250919424 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated261554024598528 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated13341775585976 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated254455180980832 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed274313744883200 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 160509257!9'%O MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated274313849352192 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed260939323362304 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed260939323362304 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13374421520896 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 _step1213 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 $ _timestamp1.785802049800905e+09 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 160518913 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed13341689782852 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 261497092 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 103405826 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 100978179 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated260939340401664 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13374508950528 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated260939340401664 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed275290250919424 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated261554024598528 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated13341775585976 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated254455180980832 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed274313744883200 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 160509257 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current254 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 100978177 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13736390287360 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 100978177 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated59844797 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed43561031 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13736251224576 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed267796853724324 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 ^c'2%- _wandbruntime6763  _runtime6763LI%M K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 100984820 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated274331895741952 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13375301140480 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 103412666 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 261514272 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 160519780 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed260956490125312 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 trainer/global_step15199 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 261514272 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 _step1214 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed43563934 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13737157991424 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 % _timestamp1.7858020501830382e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated260956507164672 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 100984822 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated261571231715328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed274331791265792 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 160529450 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed261571206811648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13375388577280 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 100984820 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current230 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 _runtime6763.006374099 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated274331895741952 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed59848732 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current232 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed275308364803072 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed254471904113792 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 261504600 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 261504600 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated13737286561280 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 103412898 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160&&%M C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated260956507164672 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 100984822 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated261571231715328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed274331791265792 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 160529450 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed261571206811648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13375388577280 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 100984820 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current230 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 _runtime6763.006374099 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated274331895741952 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed59848732 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current232 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed275308364803072 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed254471904113792 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 261504600 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 261504600 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated13737286561280 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 103412898 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 100984820 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated274331895741952 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13375301140480 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 103412666 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 261514272 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 160519780 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed260956490125312 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 trainer/global_step15199 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 261514272 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 _step1214 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed43563934 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13737157991424 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 % _timestamp1.7858020501830382e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ܗk2%- _wandbruntime6763  _runtime6763fy^ % 2loss/FBt1/assignment_loss0.005169788996359633 .loss/FBt2/detection_loss1.635066496613233 epoch0 % _timestamp1.7858020501850355e+09 1loss/SRqqt2/assignment_loss0.889041064986183 3loss/FBt2/assignment_loss8.828083987659194e-05 .loss/FRt1/detection_loss2.401274691977435 1loss/SRqqt1/detection_loss0.9017918886225901 1loss/SRqqt2/detection_loss1.0649230225632589 _runtime6763.007865075 1loss/SRqqt1/assignment_loss0.792895359688519 /loss/FRt2/detection_loss2.2940996708814057 _step1215 &loss/total_loss15.422908505683111 0loss/FRt1/assignment_loss2.6097552431954276 0loss/FRt2/assignment_loss2.1331585913896562 /loss/FBt1/detection_loss0.6956444059291648 trainer/global_step151993% &loss/total_loss15.422908505683111 0loss/FRt1/assignment_loss2.6097552431954276 0loss/FRt2/assignment_loss2.1331585913896562 /loss/FBt1/detection_loss0.6956444059291648 trainer/global_step15199 2loss/FBt1/assignment_loss0.005169788996359633 .loss/FBt2/detection_loss1.635066496613233 epoch0 % _timestamp1.7858020501850355e+09 1loss/SRqqt2/assignment_loss0.889041064986183 3loss/FBt2/assignment_loss8.828083987659194e-05 .loss/FRt1/detection_loss2.401274691977435 1loss/SRqqt1/detection_loss0.9017918886225901 1loss/SRqqt2/detection_loss1.0649230225632589 _runtime6763.007865075 1loss/SRqqt1/assignment_loss0.792895359688519 /loss/FRt2/detection_loss2.2940996708814057 _step1215 v%(: U gpu.0.gpu5.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts95.556' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.851999999999997 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0_f%(:  U disk.drbd1285.out0 disk.rbd0.out0) disk./.usagePercent32.34442845700147 disk.drbd1289.in0 disk.nvme0n1.in 4.64453125 disk.drbd1931.in0 disk.sda.in1.59375 disk.drbd2118.in0 disk.drbd1673.out0 disk.drbd1510.in0 disk.drbd1510.out0 disk.drbd1828.in0 disk.rbd1.out 27.1298828125 disk.drbd1965.out0 disk.drbd2111.out0 disk.drbd1817.in0$ disk./.usageGB578.3512306213379 disk.dm-0.in 4.64453125 disk.drbd1584.out0 disk.drbd1356.in0 disk.sda1.in0 disk.drbd2118.out0 disk.rbd1.in0 disk.drbd1550.out0 disk.drbd2078.out0 disk.drbd1817.out0 disk.drbd1931.out0 disk.drbd1552.in0 disk.drbd1938.in0 disk.drbd2111.in0 disk.drbd2078.in0 network.recv771663 disk.drbd1226.out0 disk.drbd1356.out0 disk.drbd1828.out0 disk.drbd1550.in0 disk.rbd0.in0 network.sent 1.3999279e+07 disk.dm-0.out 289.783203125 disk.sda1.out0% memory_percent41.257280111312866* proc.memory.availableMB38497.62890625$ proc.memory.rssMB11260.33203125 disk.drbd1584.in0 disk.sda2.out 183.3671875 disk.drbd2041.in0 disk.drbd1938.out0 disk.sda.out 183.3671875 cpu13.730651882264166 proc.cpu.threads29 disk.sda2.in1.59375 disk.drbd1285.in0" disk.nvme0n1.out 289.783203125) proc.memory.percent17.18190312385559 disk.drbd1226.in0 disk.drbd1965.in0 disk.drbd1673.in0 disk.drbd2041.out0 disk.drbd1552.out0 disk.drbd1289.out0%(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts96.052' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent32.01733333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0h%(:  ؿU disk.drbd2041.out0 disk.drbd1289.out0 disk.sda1.out0 disk.drbd1965.out0 disk.drbd1510.in0 disk.drbd2111.out0 proc.cpu.threads29 disk.drbd1673.out0 disk.drbd1285.out0 disk.dm-0.out 290.033203125 cpu13.730202451943649 disk.drbd1828.in0 disk.drbd1510.out0 disk.drbd2118.out0* proc.memory.percent17.182010412216187 disk.dm-0.in 4.64453125 disk.drbd1584.in0$ proc.memory.rssMB11260.40234375 disk.drbd1550.in0 disk.drbd1226.in0" disk.nvme0n1.out 290.033203125 disk.sda2.out 183.82421875 disk.drbd1817.out0 disk.drbd1931.in0 disk.sda2.in1.59375 disk.drbd2078.in0 disk.drbd2078.out0 disk.drbd1965.in0 disk.drbd2041.in0 disk.drbd1828.out0 disk.sda1.in0 disk.drbd2118.in0 network.recv773370$ disk./.usageGB578.3511924743652 disk.drbd1226.out0 disk.drbd1584.out0 disk.drbd1356.out0 disk.sda.in1.59375 disk.drbd1552.out0$ memory_percent41.25766158103943* proc.memory.availableMB38497.37890625 disk.drbd1552.in0 disk.drbd1550.out0 disk.drbd1285.in0 disk.drbd2111.in0 disk.drbd1817.in0 disk.rbd1.in0 disk.drbd1289.in0 disk.rbd0.out0 disk.drbd1356.in0 network.sent 1.4044543e+07) disk./.usagePercent32.34442632362306 disk.nvme0n1.in 4.64453125 disk.sda.out 183.82421875 disk.rbd1.out 27.2236328125 disk.drbd1673.in0 disk.drbd1931.out0 disk.drbd1938.in0 disk.drbd1938.out0 disk.rbd0.in0&2%- _wandbruntime6785  _runtime6785% & lr-AdamW/pg10.0003930000717144327 & lr-AdamW/pg20.0003930000717144327 trainer/global_step15249 % _timestamp1.7858020722669997e+09 _runtime 6785.09006932 _step1216x^% & lr-AdamW/pg20.0003930000717144327 trainer/global_step15249 % _timestamp1.7858020722669997e+09 _runtime 6785.09006932 _step1216 & lr-AdamW/pg10.0003930000717144327 2o2%- _wandbruntime6785  _runtime6785!'%O M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 101310327 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13781592497664 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated275216183638016 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 % _timestamp1.7858020722682524e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed13418417656320 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated261797678552064 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 161037259 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated60041647 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed13385578246360 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed268677750803832 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed261797661512704 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed275216079169024 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current273 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed255292172557472 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 161046915 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 103746272 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032\u%O M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 161037259 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated60041647 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed13385578246360 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed268677750803832 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed261797661512704 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 Jv#7853646316 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 262347586 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated261797678552064 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed43704354 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 262357244 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13418505085952 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13418417656320 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 161037259 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current271 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 trainer/global_step15249 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 161046915 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed275216079169024 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current273 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed255292172557472 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 161046915 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 103746272 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 101310327 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13781592497664 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated275216183638016 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 % _timestamp1.7858020722682524e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed13418417656320 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated261797678552064 F2%- _wandbruntime6785  _runtime67854&%M @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 101316972 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed60045582 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 262374424 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated261814845315072 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13419384712704 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 262374424 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 262364752 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 161047782 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current246 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated255308929769152 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated275234230027776 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated275234230027776 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed43707204 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated268695471291970 _runtime6785.483828884 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated276214076181504 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 101316970 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 101316970 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated60045584 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 101316972 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed261814828275712 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13419384712704 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated261814845315072 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed255308912729792 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13419297275904 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 161057452 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 103753034 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 trainer/global_step15249 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13782364250624 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 _step1218 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 103752786 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 161047782 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed13419297275904 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated13782488626176 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0<&%M % _timestamp1.7858020726604795e+09 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated262431587555328 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 161057452 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 149279232 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed268695368449374 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 262374424 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 262364752 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 161047782 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current246 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated255308929769152 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated275234230027776 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated275234230027776 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed43707204 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated268695471291970 _runtime6785.483828884 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated276214076181504 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 101316970 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 101316970 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated60045584 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 101316972 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed261814828275712 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13419384712704 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated261814845315072 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed255308912729792 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13419297275904 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 161057452 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 103753034 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 trainer/global_step15249 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13782364250624 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 _step1218 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 103752786 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 161047782 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed13419297275904 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated13782488626176 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 101316972 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed60045582 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 262374424 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated261814845315072 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13419384712704 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 EDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current262 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed263274711374848 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed13462413791744 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 101642477 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 161574917 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated43847872 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 166063616 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed13429466709868 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated269558750725824 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed269558647883340 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13826654349824 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13462501221376 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed277101365724672 trainer/global_step15299 _runtime6806.762683186 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated276118517923840 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated262656016702464 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13462413791744 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated276118517923840 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 161565261 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed276118413454848 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed276118413454848 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 101642479 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed60238495 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 104086369 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13826795509760 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 141159936 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated13429552512992 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated263274736278528 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 104086105 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed262655999663104 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current262 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed263274711374848 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed13462413791744 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 101642477 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 161574917 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated43847872 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 166063616 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed13429466709868 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated269558750725824 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed269558647883340 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13826654349824 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13462501221376 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed277101365724672 trainer/global_step15299 _runtime6806.762683186 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated276118517923840 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated262656016702464 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13462413791744 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated276118517923840 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 161565261 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed276118413454848 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed276118413454848 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 101642479 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed60238495 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 104086369 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13826795509760 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 141159936 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated13429552512992 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated263274736278528 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 104086105 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed262655999663104 JDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed262673166426112 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13463380848128 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 263234576 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 _step1222 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed276136459837440 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated262673183465472 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed13430344183090 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 161575784 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated277119633376768 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated60242434 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13463380848128 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 101649122 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current236 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed269576265528882 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _runtime6807.126115727 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 101649122 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated263291943395328 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated269576368371478 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated43850710 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated13430429986326 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated276136564313600 % _timestamp1.7858020943027728e+09 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed256145921345792 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 263224904 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 161585454 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated276136564313600 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed262673166426112 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 trainer/global_step15299 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 263224904 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 104092906 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13827561411584 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated256145938385152 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144yȭ%M S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed276136459837440 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated262673183465472 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed13430344183090 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 161575784 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated277119633376768 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated60242434 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13463380848128 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 101649122 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current236 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 S?DevicDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 101649122 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated263291943395328 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated269576368371478 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated43850710 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated13430429986326 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated276136564313600 % _timestamp1.7858020943027728e+09 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed256145921345792 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 263224904 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 161585454 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated276136564313600 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed262673166426112 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 trainer/global_step15299 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 263224904 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 104092906 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13827561411584 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated256145938385152 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 104093144 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed277119479903232 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed60242432 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13463293411328 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed262673166426112 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13463380848128 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 263234576 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 _step1222 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 }2%- _wandbruntime6807  _runtime68076$% 2loss/SRqqt2/assignment_loss0.8196205691354531 /loss/FRt2/detection_loss2.6485047512687743 /loss/FBt2/detection_loss0.9393429868699362 _step1223 /loss/FRt2/assignment_loss2.495115278288722 /loss/FBt1/detection_loss0.7034180424617309 &loss/total_loss15.733079695496162 4loss/FBt2/assignment_loss2.9802317058624794e-08 $ _timestamp1.785802094304548e+09 _runtime6807.127403008 /loss/FRt1/assignment_loss2.612661840964337 2loss/SRqqt1/assignment_loss0.7737860521624008 loss/FBt1/assignment_loss0 .loss/FRt1/detection_loss2.358507847451434 1loss/SRqqt1/detection_loss1.3563436083638898 1loss/SRqqt2/detection_loss1.0257786887271676 epoch0 trainer/global_step15299w% /loss/FRt1/assignment_loss2.612661840964337 2loss/SRqqt1/assignment_loss0.7737860521624008 loss/FBt1/assignment_loss0 .loss/FRt1/detection_loss2.358507847451434 1loss/SRqqt1/detection_loss1.3563436083638898 1loss/SRqqt2/detection_loss1.0257786887271676 epoch0 trainer/global_step15299 2loss/SRqqt2/assignment_loss0.8196205691354531 /loss/FRt2/detection_loss2.6485047512687743 /loss/FBt2/detection_loss0.9393429868699362 _step1223 /loss/FRt2/assignment_loss2.495115278288722 /loss/FBt1/detection_loss0.7034180424617309 &loss/total_loss15.733079695496162 4loss/FBt2/assignment_loss2.9802317058624794e-08 $ _timestamp1.785802094304548e+09 _runtime6807.127403008 %(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts121.874' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent40.62466666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0EPi%(:  U* proc.memory.availableMB38497.44921875 disk.drbd1965.out0 disk.drbd1552.in0 disk.drbd2078.out0 disk.drbd1584.in0 cpu13.72954289467612 disk.drbd2118.in0# disk.nvme0n1.out290.5771484375 disk.drbd1931.in0 disk.drbd1510.out0 disk.dm-0.out290.5771484375$ proc.memory.rssMB11260.40234375 disk.drbd1285.out0 disk.drbd1828.in0 disk.drbd1828.out0 disk.sda.out 184.42578125 disk.rbd1.out 27.2861328125 disk.drbd1510.in0 proc.cpu.threads29 disk.drbd1817.in0 disk.drbd1817.out0 network.recv775884 disk.drbd1550.in0 disk.drbd1550.out0 disk.drbd1289.in0 disk.nvme0n1.in 4.64453125) disk./.usagePercent32.34442632362306 disk.drbd2041.in0 disk.drbd2041.out0 disk.drbd1289.out0 disk.drbd1226.out0 disk.rbd1.in0 disk.sda.in1.59375 disk.sda1.in0 disk.sda2.out 184.42578125 disk.drbd2118.out0 disk.drbd1931.out0 disk.drbd1673.out0 disk.sda1.out0 disk.drbd1226.in0 disk.drbd1584.out0 disk.drbd1356.out0 disk.sda2.in1.59375 network.sent 1.4092625e+07$ disk./.usageGB578.3511924743652 disk.drbd1673.in0 disk.drbd1965.in0 disk.drbd1938.in0 disk.drbd1938.out0 disk.rbd0.out0 disk.drbd2111.in0 disk.dm-0.in 4.64453125 disk.drbd2111.out0 disk.drbd1552.out0 disk.drbd2078.in0 disk.drbd1356.in0$ memory_percent41.25755429267883 disk.rbd0.in0 disk.drbd1285.in0* proc.memory.percent17.182010412216187u! %(: U gpu.0.gpu24.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts121.365' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent40.455 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0qe%(:  U disk.drbd2041.in0 disk.drbd1226.in0 network.recv777257 disk.drbd1673.in0 disk.rbd0.out0 disk.drbd2111.in0 disk.drbd1285.in0 disk.dm-0.in 4.64453125$ proc.memory.rssMB11260.40234375 disk.drbd1550.in0$ disk.nvme0n1.out290.81591796875 disk.drbd1931.out0 disk.drbd1938.in0 disk.drbd1510.in0 disk.drbd1584.in0 disk.drbd2118.out0 disk.nvme0n1.in 4.64453125 disk.rbd1.out 27.318359375 disk.drbd1965.out0 disk.drbd1931.in0 disk.sda.in1.59375 disk.drbd2078.in0 disk.drbd1584.out0" memory_percent41.257244348526 disk.sda1.in0 disk.drbd1226.out0 cpu13.729932231297813 disk.drbd1289.in0 disk.drbd1356.out0 disk.sda2.out 184.953125 disk.rbd1.in0 proc.cpu.threads29 disk.drbd1550.out0 disk.drbd1938.out0 disk.drbd2078.out0 disk.drbd1356.in0 disk.drbd1817.out0 network.sent 1.4137659e+07 disk.drbd2041.out0 disk.drbd1285.out0 disk.sda2.in1.59375 disk.drbd1965.in0) disk./.usagePercent32.34442739031226 disk.drbd1828.in0! disk.dm-0.out290.81591796875 disk.drbd1817.in0 disk.sda.out 184.953125 disk.drbd1828.out0 disk.drbd1552.in0 disk.drbd2118.in0 disk.drbd2111.out0* proc.memory.percent17.182010412216187 disk.drbd1289.out0 disk.rbd0.in0 disk.drbd1510.out0* proc.memory.availableMB38497.65234375 disk.drbd1552.out0$ disk./.usageGB578.3512115478516 disk.drbd1673.out0 disk.sda1.out0g]2%- _wandbruntime6829  _runtime6829!& % _timestamp1.7858021163854187e+09 _runtime6829.208394763 _step1224 ' lr-AdamW/pg10.00039557729036296326 ' lr-AdamW/pg20.00039557729036296326 trainer/global_step15349& ' lr-AdamW/pg20.00039557729036296326 trainer/global_step15349 % _timestamp1.7858021163854187e+09 _runtime6829.208394763 _step1224 ' lr-AdamW/pg10.00039557729036296326 2&- _wandbruntime6829  _runtime6829Ř'&O ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 162093263 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 162102919 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed270439544962848 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 162093263 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed264135067214848 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 101974629 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 101974629 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 264067890 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed263514337813504 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 104426726 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13871837192704 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 101974627 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13506409927168 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated278007068374016 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated13473440976500 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated60435347 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated277020852209664 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed277020747740672 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 162102919 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated270439647805332 % _timestamp1.7858021163863804e+09 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated13506497356800 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13871976255488 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 104426485 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed60435345 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed263514337813504 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 trainer/global_step15349 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed13506409927168 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current239 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed278006904407552 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated277020852209664 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated43991379 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548%u.S&O VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 264067890 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed263514337813504 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 104426726 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13871837192704 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 101974627 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13506409927168 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated278007068374016 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated13473440976500 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated60435347 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated277020852209664 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed277020747740672 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 162102919 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated270439647805332 % _timestamp1.7858021163863804e+09 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated13506497356800 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13871976255488 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 104426485 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed60435345 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed263514337813504 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 trainer/global_step15349 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed13506409927168 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current239 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed278006904407552 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.alwlocated277020852209664 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated43991379 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 162093263 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 162102919 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed270439544962848 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 162093263 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed264135067214848 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 101974629 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 101974629 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ~t2&- _wandbruntime6829  _runtime6829|=7&&M OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 101981272 $ _timestamp1.785802116755518e+09 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated13474318449834 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated263531521615872 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current255 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 162113456 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed13507289546752 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated277038898599424 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 104433194 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 _step1226 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated13872874069504 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13507376983552 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed277038794123264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated60439284 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated263531521615872 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13507376983552 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed270457162608390 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated256982947001152 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current253 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed13474232646598 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 264094728 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed263531504576512 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 101981270 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated278025173304832 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed264152274331648 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated270457265450986 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated43994165 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 264085056 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed263531504576512 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 101981270 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 162103786 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed43993912 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 _runtime6829.578895711 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 162113456 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 264094728 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136&&M A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 101981270 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 162103786 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed43993912 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 _runtime6829.578895711 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 162113456 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 264094728 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 101981272 $ _timestamp1.785802116755518e+09 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated13474318449834 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated263531521615872 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current255 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 162113456 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed13507289546752 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated277038898599424 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 104433194 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 _step1226 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated13872874069504 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13507376983552 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed277038794123264 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated60439284 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated263531521615872 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13507376983552 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed270457162608390 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated256982947001152 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current253 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed13474232646598 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 264094728 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed263531504576512 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 101981270 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated278025173304832 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed264152274331648 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated270457265450986 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated43994165 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 264085056 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed263531504576512 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 w82&- _wandbruntime6829  _runtime6829x & /loss/FRt1/assignment_loss2.674474927339148 2loss/SRqqt1/assignment_loss0.5832403183421008 2loss/SRqqt2/assignment_loss0.9132919326411348 3loss/FBt2/assignment_loss7.267867539623342e-07 0loss/SRqqt2/detection_loss1.115838629246606 trainer/global_step15349 0loss/FRt2/assignment_loss1.7349385850959353 _runtime6829.580021881 .loss/FBt1/detection_loss1.030733333998605 3loss/FBt1/assignment_loss0.0015721753435699564 /loss/FRt1/detection_loss2.3073329824082394 .loss/FRt2/detection_loss2.141286471568876 % _timestamp1.7858021167571986e+09 _step1227 1loss/SRqqt1/detection_loss1.3760527722223337 /loss/FBt2/detection_loss0.7917307946650732 &loss/total_loss14.670493649658376 epoch0hZ& _runtime6829.580021881 .loss/FBt1/detection_loss1.030733333998605 3loss/FBt1/assignment_loss0.0015721753435699564 /loss/FRt1/detection_loss2.3073329824082394 .loss/FRt2/detection_loss2.141286471568876 % _timestamp1.7858021167571986e+09 _step1227 1loss/SRqqt1/detection_loss1.3760527722223337 /loss/FBt2/detection_loss0.7917307946650732 &loss/total_loss14.670493649658376 epoch0 /loss/FRt1/assignment_loss2.674474927339148 2loss/SRqqt1/assignment_loss0.5832403183421008 2loss/SRqqt2/assignment_loss0.9132919326411348 3loss/FBt2/assignment_loss7.267867539623342e-07 0loss/SRqqt2/detection_loss1.115838629246606 trainer/global_step15349 0loss/FRt2/assignment_loss1.7349385850959353 <&(: U gpu.0.gpu13.0 gpu.0.memory1, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts95.584' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.861333333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0*Qh&(:  V disk.drbd1828.in0 disk.drbd1550.out0 disk.drbd2118.in0 network.sent 1.4140718e+07 disk.drbd1356.in0 disk.drbd1817.in0 disk.drbd1931.in0 disk.drbd1584.in0$ memory_percent41.25863313674927 disk.drbd2111.out0 disk.drbd1289.out0 disk.sda1.out0 disk.drbd1938.out0! disk.dm-0.out291.08154296875 disk.drbd2078.out0 disk.drbd1584.out0$ disk.nvme0n1.out291.08154296875 disk.drbd2041.in0 disk.drbd1828.out0 disk.drbd1510.out0 disk.drbd1226.out0) proc.memory.availableMB 38496.7421875 disk.drbd1938.in0 disk.drbd2078.in0$ disk./.usageGB578.3504600524902 disk.rbd1.in0) disk./.usagePercent32.34438536275761 disk.drbd2111.in0 disk.drbd1552.out0 disk.drbd1673.in0 disk.drbd1289.in0 disk.rbd0.in0 disk.drbd2118.out0$ proc.memory.rssMB11260.40234375 network.recv778187 disk.drbd1965.in0 disk.drbd1965.out0 disk.sda1.in0 disk.drbd1285.out0 disk.drbd1510.in0 disk.nvme0n1.in 4.64453125 disk.rbd1.out 27.412109375 disk.drbd1931.out0 cpu13.729936540976857 disk.sda2.in1.59375 disk.sda2.out 185.3671875 disk.drbd1817.out0 disk.drbd1226.in0 disk.drbd1356.out0 disk.sda.out 185.3671875 disk.drbd1285.in0 disk.drbd1552.in0* proc.memory.percent17.182010412216187 disk.drbd1550.in0 disk.rbd0.out0 disk.sda.in1.59375 disk.dm-0.in 4.64453125 disk.drbd1673.out0 disk.drbd2041.out0 proc.cpu.threads29]52&- _wandbruntime6851  _runtime6851]& & lr-AdamW/pg10.0003968658996872286 & lr-AdamW/pg20.0003968658996872286 trainer/global_step15399 % _timestamp1.7858021384523404e+09 _runtime6851.275309248 _step1228,& & lr-AdamW/pg10.0003968658996872286 & lr-AdamW/pg20.0003968658996872286 trainer/global_step15399 % _timestamp1.7858021384523404e+09 _runtime6851.275309248 _step1228 u2&- _wandbruntime6851  _runtime6851a&O \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated257803215444832 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed277923082026496 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated60632197 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed13550406062592 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated13550493492224 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 102306777 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated277923186495488 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 104767504 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current270 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 % _timestamp1.7858021384532948e+09 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 102306777 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed44135039 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated264995447958528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed271320442042356 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13550493492224 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 trainer/global_step15399 _runtime6851.276576346 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 162630921 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed60632195 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed264995423054848 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated271320544884840 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 162621265 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated277923186495488 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated264372693003264 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 264937700 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 102306779 B.on_train_batch_start/active.large_pool.current2 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed264372675963904 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 104767234 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current268 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 102306779 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed277923082026496 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed257803198405472 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated13517329440008 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed13517243636884 '&O RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 264937700 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 102306779 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current268 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 102306779 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed277923082026496 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed257803198405472 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated13517329440008 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed13517243636884 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated257803215444832 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed277923082026496 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated60632197 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed13550406062592 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated13550493492224 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 102306777 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated277923186495488 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 104767504 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current270 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 % _timestamp1.7858021384532948e+09 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 102306777 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed44135039 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated264995447958528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed271320442042356 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13550493492224 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 trainer/global_step15399 _runtime6851.276576346 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 162630921 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed60632195 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed264995423054848 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated271320544884840 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 162621265 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated277923186495488 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated264372693003264 _[2&- _wandbruntime6851  _runtime6851V܀Y&&M ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13551373118976 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated271338162530494 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 264954880 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated277941232885248 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 264945208 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed278930540655104 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 _step1230 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated257819955617152 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed13551285682176 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated264389859766272 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 264954880 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime6851.644970367 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed13518121110106 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 102313422 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current249 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated277941232885248 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 162631788 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed271338059687898 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated265012655075328 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 162641458 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 trainer/global_step15399 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 104774354 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 264945208 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130R>!&M ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed278930540655104 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 _step1230 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated257819955617152 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed13551285682176 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated264389859766272 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 264954880 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime6851.644970367 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed13518121110106 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 102313422 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current249 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated277941232885248 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 162631788 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed271338059687898 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated265012655075328 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 162641458 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 trainer/global_step15399 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 104774354 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 264945208 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13551373118976 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated271338162530494 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 264954880 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated277941232885248 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 264945208 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 2&- _wandbruntime6851  _runtime6851& /loss/FBt2/detection_loss0.8702694367686356 trainer/global_step15399 % _timestamp1.7858021388233736e+09 0loss/FRt1/assignment_loss2.4079391154504957 2loss/SRqqt1/assignment_loss1.0362697952524929 2loss/SRqqt2/assignment_loss0.8682751281865007 -loss/FBt1/detection_loss1.56676689028326 _runtime 6851.64646875 _step1231 0loss/FRt2/assignment_loss2.1988397247734524 loss/FBt1/assignment_loss0 /loss/FRt2/detection_loss2.3841132720311484 %loss/total_loss16.02333906656778 epoch0 .loss/FRt1/detection_loss2.523057925381831 1loss/SRqqt2/detection_loss1.0788810599203174 3loss/FBt2/assignment_loss3.139074308933232e-06 0loss/SRqqt1/detection_loss1.088923579445336uf& 3loss/FBt2/assignment_loss3.139074308933232e-06 0loss/SRqqt1/detection_loss1.088923579445336 /loss/FBt2/detection_loss0.8702694367686356 trainer/global_step15399 % _timestamp1.7858021388233736e+09 0loss/FRt1/assignment_loss2.4079391154504957 2loss/SRqqt1/assignment_loss1.0362697952524929 2loss/SRqqt2/assignment_loss0.8682751281865007 -loss/FBt1/detection_loss1.56676689028326 _runtime 6851.64646875 _step1231 0loss/FRt2/assignment_loss2.1988397247734524 loss/FBt1/assignment_loss0 /loss/FRt2/detection_loss2.3841132720311484 %loss/total_loss16.02333906656778 epoch0 .loss/FRt1/detection_loss2.523057925381831 1loss/SRqqt2/detection_loss1.0788810599203174 f&(: ٺU gpu.0.gpu27.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts130.867' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent43.62233333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0t7i&(:  尃V disk.drbd1673.out0 disk.drbd1552.out0 disk.drbd1584.out0 disk.drbd2111.in0# memory_percent41.2589430809021 cpu13.729584463712794 network.recv779828 disk.drbd1356.in0 disk.rbd1.out 27.474609375 disk.drbd1550.out0 disk.drbd1285.in0 disk.drbd1965.out0 disk.drbd1552.in0$ proc.memory.rssMB11260.40234375 disk.drbd1550.in0 disk.sda1.out0 disk.dm-0.in 4.64453125 disk.drbd2078.out0 disk.drbd1828.in0* proc.memory.percent17.182010412216187 disk.drbd1289.in0 disk.sda2.in1.59375 disk.nvme0n1.in 4.64453125$ disk.nvme0n1.out291.30029296875 disk.sda1.in0 disk.drbd1285.out0 disk.drbd2111.out0 disk.drbd1673.in0 disk.drbd1931.out0) disk./.usagePercent32.34443613716374 disk.drbd1938.in0 disk.drbd1226.in0 disk.drbd1226.out0 disk.drbd2078.in0 disk.drbd2041.out0 proc.cpu.threads29 disk.sda.in1.59375 disk.drbd1510.in0 disk.drbd1510.out0 disk.drbd2118.out0) proc.memory.availableMB 38496.5390625$ disk./.usageGB578.3513679504395 disk.sda2.out 185.77734375 network.sent 1.4185727e+07 disk.drbd1289.out0 disk.rbd0.in0! disk.dm-0.out291.30029296875 disk.drbd2118.in0 disk.drbd1356.out0 disk.rbd1.in0 disk.drbd1965.in0 disk.drbd1828.out0 disk.drbd1938.out0 disk.rbd0.out0 disk.sda.out 185.77734375 disk.drbd1584.in0 disk.drbd1817.in0 disk.drbd1817.out0 disk.drbd1931.in0 disk.drbd2041.in0޺uY&(: ͢U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts120.239' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent40.07966666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors05Wf&(:  U disk.drbd1828.out0$ proc.memory.rssMB11260.40234375 disk.drbd1289.in0 disk.drbd1510.out0 disk.drbd2118.in0 network.sent 1.4230817e+07 disk.drbd2078.out0 disk.drbd1931.in0 disk.drbd1552.out0 disk.drbd1510.in0$ memory_percent41.25933647155762) disk./.usagePercent32.34443827054215 disk.sda2.in1.59375 disk.drbd2111.out0 disk.drbd1673.in0 disk.drbd1285.out0$ disk./.usageGB578.3514060974121$ disk.nvme0n1.out291.54248046875 disk.drbd1356.in0 disk.sda2.out 186.2890625 disk.drbd1673.out0 disk.rbd0.in0 disk.rbd0.out0 proc.cpu.threads29 network.recv781603 disk.drbd1584.in0 disk.drbd1356.out0 disk.drbd2041.out0' proc.memory.availableMB 38496.28125 disk.drbd1965.out0 disk.rbd1.in0 disk.rbd1.out 27.474609375 disk.drbd1226.in0 disk.drbd1226.out0 disk.drbd1550.out0 disk.drbd2118.out0 disk.drbd1552.in0 disk.drbd1938.out0 disk.drbd1285.in0 disk.nvme0n1.in 4.64453125 disk.drbd1817.out0 disk.drbd1931.out0* proc.memory.percent17.182010412216187 disk.dm-0.in 4.64453125 disk.drbd1817.in0 disk.drbd1965.in0 disk.drbd1584.out0! disk.dm-0.out291.54248046875 disk.drbd2111.in0 disk.sda.in1.59375 cpu13.729197033249388 disk.drbd2078.in0 disk.sda1.in0 disk.drbd1550.in0 disk.drbd1289.out0 disk.drbd1938.in0 disk.sda.out 186.2890625 disk.sda1.out0 disk.drbd2041.in0 disk.drbd1828.in02&- _wandbruntime6873  _runtime6873C& ' lr-AdamW/pg10.00039815450901149393 ' lr-AdamW/pg20.00039815450901149393 trainer/global_step15449 $ _timestamp1.785802160467649e+09 _runtime6873.290618872 _step1232,& ' lr-AdamW/pg10.00039815450901149393 ' lr-AdamW/pg20.00039815450901149393 trainer/global_step15449 $ _timestamp1.785802160467649e+09 _runtime6873.290618872 _step1232 cE2&- _wandbruntime6873  _runtime6873-SI'&O M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 102638927 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated44280103 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated278825520781312 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated13594489627648 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 102638927 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated272201441964348 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 102638929 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 105109150 _runtime6873.291831051 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 265788194 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13594402198016 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed265231014114304 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed278825416312320 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 trainer/global_step15449 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 163158923 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current261 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed279818003241984 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated258640224060832 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed272201339121864 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 163149267 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated265231031153664 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated13561217903516 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated60829047 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed258640207021472 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed60829045 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed44279842 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 163158923 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 $ _timestamp1.785802160468573e+09 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated265231031153664 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 265797852 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13962352924160 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 163149267 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13962224347136 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 102638929 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13594489627648 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated265855803798528 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed13561132100392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed278825416312320 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 105108887 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 _step1233 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current263 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed265855778894848 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed13594402198016 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed265231014114304 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1Ȝ'&O QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current261 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed279818003241984 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated258640224060832 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed272201339121864 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 163149267 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated265231031153664 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated13561217903516 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated60829047 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed258640207021472 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed60829045 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed44279842 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 163158923 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 $ _timestamp1.785802160468573e+09 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated265231031153664 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 265797852 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated13962352924160 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 163149267 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed13962224347136 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 102638929 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13594489627648 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated265855803798528 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed13561132100392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed278825416312320 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 105108887 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 _step1233 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current263 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed265855778894848 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed13594402198016 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed265231014114304 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 102638927 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated44280103 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated278825520781312 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated13594489627648 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 102638927 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated272201441964348 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 102638929 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 105109150 _runtime6873.291831051 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 265788194 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13594402198016 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed265231014114304 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed278825416312320 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 trainer/global_step15449 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 163158923 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 2&- _wandbruntime6873  _runtime6873H> &M A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 163169460 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 105115679 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 trainer/global_step15449 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13595281817600 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 102645572 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated60832984 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed265872986011648 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed13562009573614 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 265815032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed265248180877312 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 163169460 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated265248197916672 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 265805360 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed44282697 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed278843462694912 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 _runtime6873.660064705 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated13963259831296 @:DeviceC StatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated265248197916672 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current252 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated265873010915328 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 163159790 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13595369254400 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 102645570 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 _step1234 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13963127067136 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 265805360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated13562095376850 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 102645572 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 102645570 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 265815032 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 105115931 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current250 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed279836113078784 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 % _timestamp1.7858021608367155e+09 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated272219059610002 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated278843567171072 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed60832982 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888j4h&&M WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed265248180877312 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13595369254400 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated44282947 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 265805360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated13562095376850 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 102645572 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 102645570 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 265815032 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 105115931 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current250 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed279836113078784 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 % _timestamp1.7858021608367155e+09 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated272219059610002 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated278843567171072 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed60832982 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 163169460 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 105115679 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 trainer/global_step15449 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13595281817600 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 102645572 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated60832984 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed265872986011648 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed13562009573614 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 265815032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed265248180877312 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 163169460 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated265248197916672 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 265805360 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed44282697 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed278843462694912 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 _runtime6873.660064705 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated13963259831296 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated265248197916672 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current252 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated265873010915328 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 163159790 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13595369254400 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 102645570 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 _step1234 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed13963127067136 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 EDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated44423892 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed266089352264704 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed279727750598144 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 trainer/global_step15499 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 102971079 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 163677269 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated279727855067136 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 % _timestamp1.7858021827551398e+09 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated266089369304064 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 163686925 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed61025895 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated266089369304064 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 266648346 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated266716159638528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed279727750598144 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated279727855067136 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed266089352264704 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 102971077 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13638398333440 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 163686925 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated259477232676832 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current281 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed266716134734848 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14007431988736 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated280723724398592 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 105449789 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed280723566723584 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated273082339043856 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed273082236201372 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 % _timestamp1.7858021827551398e+09 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated266089369304064 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 163686925 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed61025895 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated266089369304064 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 266648346 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated266716159638528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed279727750598144 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated279727855067136 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed266089352264704 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 102971077 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13638398333440 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 163686925 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated259477232676832 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current281 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed266716134734848 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14007431988736 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated280723724398592 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 105449789 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed280723566723584 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated273082339043856 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed273082236201372 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated44423892 "2&- _wandbruntime6895  _runtime6895I[&&M _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed266733341851648 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated14008467106304 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 163697462 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 266675184 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 163687792 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 102977720 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed13605898037122 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed273099853846914 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 _step1238 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed13639277953024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated44426716 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated280741833861632 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed259493955809792 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 _runtime6895.958245095 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 266675184 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 163697462 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed279745796980736 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 105456303 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current245 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated266106536067072 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed266106519027712 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13639365389824 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 % _timestamp1.7858021831348503e+09 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13639365389824 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13639277953024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed44426471 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated13605983840358 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 102977722 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed266106519027712 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13639365389824 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 % _timestamp1.7858021831348503e+09 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13639365389824 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13639277953024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed44426471 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated13605983840358 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 102977722 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 163697462 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 266675184 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 163687792 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 102977720 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed13605898037122 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed273099853846914 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 _step1238 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed13639277953024 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated44426716 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated280741833861632 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed259493955809792 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 _runtime6895.958245095 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 266675184 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 163697462 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed279745796980736 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 105456303 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current245 y2&- _wandbruntime6895  _runtime6895iu & 1loss/SRqqt1/detection_loss1.0616532310976514 /loss/FRt2/assignment_loss2.328686768100375 2loss/SRqqt2/assignment_loss0.9288580509056041 3loss/FBt1/assignment_loss0.0004729498177715124 0loss/SRqqt2/detection_loss1.104483749192681 /loss/FBt1/detection_loss1.1470621630168436 /loss/FBt2/detection_loss0.8153926833532751 trainer/global_step15499 0loss/FRt1/assignment_loss2.3297749501008256 3loss/FBt2/assignment_loss0.0010763200931165784 /loss/FRt1/detection_loss2.8880340792238712 _runtime 6895.95936774 2loss/SRqqt1/assignment_loss0.4693897614987301 .loss/FRt2/detection_loss2.394767154274242 &loss/total_loss15.469651860674988 epoch0 $ _timestamp1.785802183136545e+09 _step1239$S& trainer/global_step15499 0loss/FRt1/assignment_loss2.3297749501008256 3loss/FBt2/assignment_loss0.0010763200931165784 /loss/FRt1/detection_loss2.8880340792238712 _runtime 6895.95936774 2loss/SRqqt1/assignment_loss0.4693897614987301 .loss/FRt2/detection_loss2.394767154274242 &loss/total_loss15.469651860674988 epoch0 $ _timestamp1.785802183136545e+09 _step1239 1loss/SRqqt1/detection_loss1.0616532310976514 /loss/FRt2/assignment_loss2.328686768100375 2loss/SRqqt2/assignment_loss0.9288580509056041 3loss/FBt1/assignment_loss0.0004729498177715124 0loss/SRqqt2/detection_loss1.104483749192681 /loss/FBt1/detection_loss1.1470621630168436 /loss/FBt2/detection_loss0.8153926833532751 pg&(:  U disk.drbd1510.out0 disk.drbd2078.out0 disk.drbd2118.out0 disk.drbd2111.in0 disk.drbd1285.in0$ proc.memory.rssMB11260.40234375 network.recv784143 disk.drbd2041.out0 disk.rbd0.in0 disk.drbd1965.out0 disk.drbd1550.in0 disk.sda1.out0 disk.drbd1584.out0 disk.drbd1828.in0 disk.drbd1673.in0 disk.drbd2041.in0$ memory_percent41.25949144363403 disk.drbd1931.out0 disk.drbd1938.out0 disk.sda2.in1.59375* proc.memory.percent17.182010412216187 disk.drbd1289.in0 disk.sda.in1.59375 disk.drbd1285.out0 disk.rbd1.in0 cpu13.728461096991191 disk.drbd1226.out0 disk.drbd1817.out0 disk.drbd1552.in0 disk.drbd1938.in0 disk.drbd2118.in0 disk.nvme0n1.in 4.64453125 disk.drbd1356.in0 disk.drbd2111.out0) proc.memory.availableMB 38496.1796875 disk.drbd1289.out0 disk.drbd1584.in0 disk.drbd1817.in0 disk.drbd1931.in0 disk.drbd1673.out0 disk.drbd1356.out0 network.sent 1.4278939e+07$ disk./.usageGB578.3514060974121$ disk.nvme0n1.out292.49951171875 proc.cpu.threads29 disk.drbd1550.out0 disk.sda.out 187.265625! disk.dm-0.out292.49951171875 disk.drbd1226.in0 disk.drbd2078.in0 disk.sda2.out 187.265625 disk.dm-0.in 4.64453125 disk.rbd1.out 27.7802734375 disk.sda1.in0) disk./.usagePercent32.34443827054215 disk.drbd1965.in0 disk.drbd1552.out0 disk.drbd1828.out0 disk.rbd0.out0 disk.drbd1510.in0Zٙ&(: ƥU gpu.0.gpu0.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts95.75' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.916666666666664 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0[&(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts97.715' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent32.571666666666665 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0ZG&(:  U disk.sda2.in1.59375 network.sent 1.4323976e+07 disk.drbd2078.out0 disk.drbd1584.in0 disk.drbd1356.out0 disk.drbd1817.in0 disk.sda1.out0 disk.rbd1.in0 proc.cpu.threads29 disk.drbd1550.in0 disk.drbd1550.out0 disk.drbd1289.in0 disk.drbd1289.out0 disk.drbd2041.in0 disk.drbd1285.out0 disk.drbd2078.in0 disk.drbd2111.in0 disk.rbd0.out0$ disk./.usageGB578.3514289855957 disk.sda2.out 187.59765625 disk.drbd1965.out0 disk.drbd1828.in0* proc.memory.availableMB38496.66796875) disk./.usagePercent32.34443955056919# disk.nvme0n1.out293.0888671875 disk.drbd1817.out0 disk.rbd1.out 27.8427734375 disk.drbd1673.in0 disk.drbd1552.out0 disk.drbd1828.out0 disk.sda.in1.59375 disk.drbd2118.in0 disk.drbd1673.out0* proc.memory.percent17.182010412216187 disk.drbd1931.out0 disk.drbd1510.in0 disk.drbd1510.out0 disk.dm-0.out293.0888671875 disk.drbd1226.out0 disk.drbd1931.in0 disk.drbd1226.in0 disk.drbd2041.out0 disk.drbd1552.in0$ memory_percent41.25874638557434 disk.dm-0.in 4.64453125 disk.drbd1356.in0 network.recv785698 disk.sda.out 187.59765625 disk.rbd0.in0 cpu13.728131239559698 disk.drbd1285.in0 disk.drbd2118.out0 disk.sda4|*1.in0 disk.drbd1938.in0 disk.drbd1938.out0$ proc.memory.rssMB11260.40234375 disk.drbd1584.out0 disk.nvme0n1.in 4.64453125 disk.drbd2111.out0 disk.drbd1965.in0r&2&- _wandbruntime6917  _runtime6917J& trainer/global_step15549 % _timestamp1.7858022047784662e+09 _runtime6917.601427658 _step1240 & lr-AdamW/pg10.0004007317276600245 & lr-AdamW/pg20.0004007317276600245=m& & lr-AdamW/pg10.0004007317276600245 & lr-AdamW/pg20.0004007317276600245 trainer/global_step15549 % _timestamp1.7858022047784662e+09 _runtime6917.601427658 _step1240 "2&- _wandbruntime6917  _runtime69179ݲ'&O N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated273963236123364 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated281629240188928 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed260314224253472 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed13648909027408 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13682481898496 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed280630084883968 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated266947707454464 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current244 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated260314241292832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 % _timestamp1.7858022047794375e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed281629084611072 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 267508498 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated13682481898496 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed266947690415104 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 267518156 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed280630084883968 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 105791022 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed61222745 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 164205271 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated267576515478528 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed13682394468864 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated61222747 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated13648994830532 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 trainer/global_step15549 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 103303229 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated280630189352960 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 164205271 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0GU'&O JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current246 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated44568521 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated266947707454464 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14052594036224 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 103303227 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 105791268 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 103303227 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 _step1241 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 103303229 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated14052724710400 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 _runtime 6917.60402104 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed273963133280880 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed267576490574848 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 164214927 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated280630189352960 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed266947690415104 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13682394468864 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed280630084883968 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 105791022 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed61222745 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 164205271 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated267576515478528 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed13682394468864 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated61222747 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated13648994830532 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 trainer/global_step15549 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 103303229 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated280630189352960 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 164205271 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated273963236123364 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated281629240188928 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed260314224253472 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed13648909027408 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13682481898496 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed280630084883968 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated266947707454464 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current244 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated260314241292832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 % _timestamp1.7858022047794375e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed281629084611072 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 267508498 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated13682481898496 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed266947690415104 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 267518156 JDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13683361525248 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 103309872 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 267535336 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated280648235742720 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed273980750926422 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14053489244160 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 105797796 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated280648235742720 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 267525664 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 103309872 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 _step1242 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 267535336 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed44571114 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed260330964425792 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 103309870 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated266964874217472 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed266964857178112 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current271 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed280648131266560 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed267593697691648 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated260330981465152 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 105798069 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated44571385 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated14053632494080 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13683274088448 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current273 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 164225464 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 $ _timestamp1.785802205153083e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed61226682 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed266964857178112*6&M PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed61226682 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed266964857178112 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 164215794 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13683361525248 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 103309872 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 267535336 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated280648235742720 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed273980750926422 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14053489244160 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 105797796 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated280648235742720 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 267525664 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 103309872 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 _step1242 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 267535336 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed44571114 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed260330964425792 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 103309870 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated266964874217472 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed266964857178112 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current271 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed280648131266560 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed267593697691648 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated260330981465152 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 105798069 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated44571385 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated14053632494080 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13683274088448 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current273 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 164225464 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 $ _timestamp1.785802205153083e+09 f2&- _wandbruntime6917  _runtime6917& 2loss/SRqqt1/assignment_loss0.5613285331192481 2loss/FBt2/assignment_loss3.81784236645899e-05 .loss/FBt2/detection_loss1.300084631703794 trainer/global_step15549 _runtime6917.978226472 0loss/FRt1/assignment_loss2.6136693796892274 loss/FBt1/assignment_loss0 .loss/FRt1/detection_loss2.529185685960726 1loss/SRqqt1/detection_loss0.9890899887472829 1loss/SRqqt2/detection_loss1.3055568562420317 /loss/FBt1/detection_loss0.3544922163889099 /loss/FRt2/detection_loss2.4500202270348868 &loss/total_loss15.496336805134444 epoch0 _step1243 2loss/SRqqt2/assignment_loss1.0534548018058407 $ _timestamp1.785802205155067e+09 0loss/FRt2/assignment_loss2.3394163060188293-z& .loss/FRt1/detection_loss2.529185685960726 1loss/SRqqt1/detection_loss0.9890899887472829 1loss/SRqqt2/detection_loss1.3055568562420317 /loss/FBt1/detection_loss0.3544922163889099 /loss/FRt2/detection_loss2.4500202270348868 &loss/total_loss15.496336805134444 epoch0 _step1243 2loss/SRqqt2/assignment_loss1.0534548018058407 $ _timestamp1.785802205155067e+09 0loss/FRt2/assignment_loss2.3394163060188293 2loss/SRqqt1/assignment_loss0.5613285331192481 2loss/FBt2/assignment_loss3.81784236645899e-05 .loss/FBt2/detection_loss1.300084631703794 trainer/global_step15549 _runtime6917.978226472 0loss/FRt1/assignment_loss2.6136693796892274 loss/FBt1/assignment_loss0 =c&(:  U disk.sda.out 187.90625 disk.drbd1510.in0 disk.sda2.in1.59375 disk.drbd2111.out0 proc.cpu.threads29 network.sent 1.4327012e+07 disk.drbd1931.in0 disk.drbd1226.in0 disk.drbd2118.in0 disk.sda2.out 187.90625 disk.drbd1817.in0 disk.drbd2078.in0 disk.drbd2078.out0$ memory_percent41.25899076461792$ proc.memory.rssMB11260.40234375 disk.drbd1289.in0 disk.drbd1938.out0 disk.rbd0.out0 disk.drbd1817.out0 disk.drbd1965.in0 cpu13.727642351245668 disk.rbd0.in0 disk.sda1.in0 disk.drbd1285.in0 disk.drbd1584.in0 disk.drbd1584.out0 disk.drbd1356.out0 disk.drbd1931.out0 disk.drbd2041.in0 disk.drbd1552.out0 disk.drbd1828.out0 disk.drbd1550.out0 disk.drbd2111.in0* proc.memory.percent17.182010412216187 disk.drbd2118.out0 disk.drbd1673.out0$ disk./.usageGB578.3513679504395 disk.drbd1550.in0 disk.drbd1226.out0 disk.rbd1.out 27.8427734375) proc.memory.availableMB 38496.5078125 disk.sda1.out0 disk.dm-0.out293.3818359375 disk.nvme0n1.in 4.64453125 disk.drbd1965.out0 disk.rbd1.in0 disk.sda.in1.59375 disk.drbd1285.out0 disk.drbd1510.out0 network.recv786677 disk.drbd1938.in0 disk.dm-0.in 4.64453125 disk.drbd2041.out0 disk.drbd1828.in0 disk.drbd1289.out0# disk.nvme0n1.out293.3818359375 disk.drbd1356.in0) disk./.usagePercent32.34443613716374 disk.drbd1673.in0 disk.drbd1552.in0Onl&(: U gpu.0.gpu33.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts123.367' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent41.12233333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0{b'2&- _wandbruntime6939  _runtime6939w,& ' lr-AdamW/pg10.00040202033698428987 ' lr-AdamW/pg20.00040202033698428987 trainer/global_step15599 # _timestamp1.78580222680848e+09 _runtime6939.631445001 _step1244p& ' lr-AdamW/pg20.00040202033698428987 trainer/global_step15599 # _timestamp1.78580222680848e+09 _runtime6939.631445001 _step1244 ' lr-AdamW/pg10.00040202033698428987 R2&- _wandbruntime6939  _runtime6939@'&O G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed13692797490916 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 trainer/global_step15599 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 164733273 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 _runtime6939.632643391 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated281532523638784 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 164742929 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed44713305 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 164733273 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 106132900 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated282534743693312 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13726478033920 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated44713547 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed261151232869472 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed267806028565504 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed281532419169792 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed281532419169792 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 103635377 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated268436871318528 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 106133144 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current244 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 _step1245 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated267806045604864 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 268378308 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed13726390604288 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated14097872374784 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 103635379 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 % _timestamp1.7858022268094304e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 103635379 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 268368650 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated2678060456048643&&O XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated61419597 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14097743797760 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated274844133202872 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed61419595 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed274844030360388 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed267806028565504 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 164742929 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 106133144 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current244 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 _step1245 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated267806045604864 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 268378308 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed13726390604288 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated14097872374784 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 103635379 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 % _timestamp1.7858022268094304e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 103635379 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 268368650 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated267806045604864 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed13692797490916 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 trainer/global_step15599 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 164733273 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 _runtime6939.632643391 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated281532523638784 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 164742929 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed44713305 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 164733273 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 106132900 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated282534743693312 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13726478033920 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated44713547 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed261151232869472 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed267806028565504 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed281532419169792 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed281532419169792 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 103635377 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated268436871318528 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 J6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated281550570028544 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed281550465552384 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 trainer/global_step15599 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 268385816 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated13693760767374 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 268395488 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 106139693 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated14098781510144 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 268395488 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated44716410 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated268454078435328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated267823212367872 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed267823195328512 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 _runtime6939.999519995 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 103642022 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed282552700180480 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed261167973041792 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 164743796 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 103642020 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 164743796 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated261167990081152 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated267823212367872 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13727357660672 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 103642022 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed44716161 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 103642020 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13727270223872 % _timestamp1.7858022271761699e+09 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated61423534 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated282552859945472 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 164753466 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 _step1246 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13727357660672 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed267823195328512 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed13727270223872 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current249 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed268454053531648 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 106139944 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14098646648832 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 _runtime6939.999519995 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 103642022 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed282552700180480 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed261167973041792 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 164743796 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 103642020 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 164743796 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated261167990081152 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated267823212367872 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13727357660672 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 103642022 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed44716161 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 103642020 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13727270223872 % _timestamp1.7858022271761699e+09 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated61423534 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated282552859945472 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 164753466 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 _step1246 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13727357660672 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed267823195328512 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed13727270223872 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current249 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed268454053531648 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 106139944 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14098646648832 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated281550570028544 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed281550465552384 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 trainer/global_step15599 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 268385816 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated13693760767374 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 268395488 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 106139693 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated14098781510144 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 268395488 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated44716410 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated268454078435328 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 Y2&- _wandbruntime6940  _runtime6940GΗ & /loss/FRt2/assignment_loss2.183612386647024 0loss/SRqqt2/assignment_loss0.96645696897977 /loss/FRt1/detection_loss2.5155152544567856 &loss/total_loss15.602033214197014 epoch0 _step1247 0loss/FRt1/assignment_loss2.7087969870506963 2loss/SRqqt1/assignment_loss0.7797990005500125 3loss/FBt2/assignment_loss9.988542145104399e-06 0loss/SRqqt1/detection_loss1.021466230458923 /loss/FBt2/detection_loss0.9856902386372288 trainer/global_step15599 1loss/SRqqt2/detection_loss0.9816479737541282 _runtime6940.000647078 2loss/FBt1/assignment_loss0.007584078274555495 .loss/FRt2/detection_loss2.447016008884499 .loss/FBt1/detection_loss1.004438097961247 % _timestamp1.7858022271778214e+09ZŇ & 1loss/SRqqt2/detection_loss0.9816479737541282 _runtime6940.000647078 2loss/FBt1/assignment_loss0.007584078274555495 .loss/FRt2/detection_loss2.447016008884499 .loss/FBt1/detection_loss1.004438097961247 % _timestamp1.7858022271778214e+09 /loss/FRt2/assignment_loss2.183612386647024 0loss/SRqqt2/assignment_loss0.96645696897977 /loss/FRt1/detection_loss2.5155152544567856 &loss/total_loss15.602033214197014 epoch0 _step1247 0loss/FRt1/assignment_loss2.7087969870506963 2loss/SRqqt1/assignment_loss0.7797990005500125 3loss/FBt2/assignment_loss9.988542145104399e-06 0loss/SRqqt1/detection_loss1.021466230458923 /loss/FBt2/detection_loss0.9856902386372288 trainer/global_step15599 4&(: ׽U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp40.0 gpu.0.powerWatts128.06' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.68666666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0㆖`&(:  ňV disk.drbd1965.in0 disk.rbd0.in0 disk.sda1.out0 disk.drbd1285.in0 disk.drbd1673.in0* proc.memory.availableMB38496.22265625 proc.cpu.threads29 disk.drbd1938.in0 disk.drbd2078.in0 disk.drbd1356.in0 disk.dm-0.in 4.64453125 disk.drbd1226.in0 disk.drbd1584.out0 disk.drbd1673.out0 network.sent 1.4372076e+07 disk.drbd1550.in0 disk.drbd1817.in0 disk.rbd1.in0* proc.memory.percent17.182016372680664 disk.drbd1289.out0 disk.rbd0.out0 disk.drbd1285.out0$ disk.nvme0n1.out293.64306640625 disk.rbd1.out27.875 disk.drbd1552.out0 disk.drbd1828.in0 disk.sda1.in0 disk.drbd2111.out0$ disk./.usageGB578.3515090942383 disk.drbd1938.out0 disk.drbd2118.out0! proc.memory.rssMB 11260.40625 disk.drbd2118.in0 disk.drbd1356.out0$ memory_percent41.25942587852478) disk./.usagePercent32.34444403066385 disk.drbd1226.out0 disk.nvme0n1.in 4.64453125 disk.drbd1931.in0 disk.drbd2041.in0 network.recv788410 disk.drbd1817.out0 disk.drbd2041.out0 disk.drbd1550.out0 disk.sda2.in1.59375 disk.drbd1965.out0 disk.drbd1931.out0 cpu13.727439594223775! disk.dm-0.out293.64306640625 disk.drbd1828.out0 disk.drbd1289.in0 disk.sda.in1.59375 disk.drbd1510.in0 disk.sda2.out 188.1953125 disk.drbd2111.in0 disk.drbd1552.in0 disk.sda.out 188.1953125 disk.drbd1510.out0 disk.drbd2078.out0 disk.drbd1584.in08̵m&(: ӤU gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp40.0 gpu.0.powerWatts125.867' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent41.955666666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0{|Xb&(:  U disk.sda.in1.59375 disk.drbd2118.in0! proc.memory.rssMB 11260.40625 disk.sda2.in1.59375) disk./.usagePercent32.34444189728544 disk.drbd2111.in0 disk.drbd1289.in0 disk.drbd1965.out0 disk.drbd1552.out0 disk.drbd1550.in0 disk.sda1.in0 disk.drbd2118.out0 disk.sda2.out 188.62109375 disk.drbd1550.out0 disk.drbd1226.out0 disk.drbd1584.out0 disk.rbd1.out27.96875 disk.drbd1673.in0 disk.drbd1552.in0 disk.drbd1828.out0 disk.rbd0.out0 disk.sda.out 188.62109375 disk.drbd1285.out0 disk.drbd1584.in0$ disk./.usageGB578.3514709472656 disk.drbd2111.out0 disk.drbd1817.out0 disk.drbd2078.in0 disk.drbd1356.out0# memory_percent41.2594199180603 proc.cpu.threads29 disk.drbd1817.in0 disk.drbd1828.in0 disk.sda1.out0 disk.dm-0.in 4.64453125! disk.dm-0.out293.86181640625 disk.drbd1931.in0 disk.drbd1931.out0 disk.drbd1938.out0 disk.rbd0.in0 disk.drbd1226.in0 disk.drbd2078.out0) proc.memory.availableMB 38496.2265625 network.recv789852 disk.rbd1.in0 disk.drbd1289.out0 disk.drbd1938.in0 disk.drbd1356.in0* proc.memory.percent17.182016372680664 cpu13.727150773911896 network.sent 1.4417332e+07 disk.drbd2041.out0 disk.drbd1965.in0 disk.drbd1673.out0 disk.drbd2041.in0 disk.nvme0n1.in 4.64453125 disk.drbd1285.in0 disk.drbd1510.in0 disk.drbd1510.out0$ disk.nvme0n1.out293.86181640625v2&- _wandbruntime6961  _runtime6961V& _step1248 ' lr-AdamW/pg10.00040330894630855507 ' lr-AdamW/pg20.00040330894630855507 trainer/global_step15649 % _timestamp1.7858022486772556e+09 _runtime 6961.50021836iA& ' lr-AdamW/pg10.00040330894630855507 ' lr-AdamW/pg20.00040330894630855507 trainer/global_step15649 % _timestamp1.7858022486772556e+09 _runtime 6961.50021836 _step1248 z g2&- _wandbruntime6961  _runtime6961zh&O N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed282434753455616 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed268664366715904 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed44858427 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 % _timestamp1.7858022486781988e+09 _runtime6961.501460437 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated268664383755264 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 269238460 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated261988258524832 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 103967529 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 165261275 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current243 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated275725030282380 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 106474872 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 103967529 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated14143031403008 _step1249 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current241 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 165261275 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated283440258561536 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13770474169344 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13770386739712 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 165270931 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14142902825984 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed13736685954424 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed13770386739712 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed275724927439896 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 O@DeviceStatsMonitor.on_train_bVatch_start/allocated_bytes.all.peak 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated61616447 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated282434857924608 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated13770474169344 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated13736771757548 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated268664383755264 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed61616445 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 165270931 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 trainer/global_step15649 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed282434753455616 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed268664366715904 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed44858427 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 % _timestamp1.7858022486781988e+09 _runtime6961.501460437 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated268664383755264 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 269238460 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated261988258524832 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 103967529 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 165261275 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current243 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated275725030282380 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 106474872 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 103967529 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated14143031403008 _step1249 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current241 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 165261275 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated283440258561536 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13770474169344 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13770386739712 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 165270931 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14142902825984 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed13736685954424 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed13770386739712 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed275724927439896 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated61616447 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated282434857924608 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated13770474169344 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated13736771757548 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated268664383755264 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 }2&- _wandbruntime6961  _runtime6961gZe&&M E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 165281468 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 trainer/global_step15649 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed283458223830528 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed13737563427646 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed262004981657792 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13771353796096 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated262004998697152 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated283458383595520 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 165271798 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 103974172 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 106482003 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 165271798 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed282452799838208 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated268681550518272 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 103974170 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed44861360 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 % _timestamp1.7858022490519319e+09 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 269255640 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 103974170 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 269245968 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed13771266359296 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 269255640 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated13737649230882 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13771266359296 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated282452904314368 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed268681533478912 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13771353796096 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed268681533478912 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated14143949320192 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated275742647928034 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 106481742 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed269314409371648 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 103974172 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed61620382 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed275742545085438 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated61620384 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 165281468 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated282452904314368 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14143814458880 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated268681550518272 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated269314434275328 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 _runtime6961.875277445 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 165271798 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 103974172 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 106482003 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 165271798 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed282452799838208 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated268681550518272 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 103974170 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed44861360 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 % _timestamp1.7858022490519319e+09 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 269255640 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 103974170 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 269245968 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed13771266359296 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 269255640 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated13737649230882 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13771266359296 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated282452904314368 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed268681533478912 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13771353796096 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed268681533478912 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated14143949320192 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated275742647928034 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 106481742 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed269314409371648 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 103974172 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed61620382 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed275742545085438 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated61620384 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 165281468 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated282452904314368 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14143814458880 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated268681550518272 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated269314434275328 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 _runtime6961.875277445 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed262004981657792 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13771353796096 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated262004998697152 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated283458383595520 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 J2&- _wandbruntime6961  _runtime6961H & 0loss/FRt1/assignment_loss2.2085043317393254 0loss/FRt2/assignment_loss2.1128393198237005 2loss/SRqqt1/assignment_loss0.8138293691102744 .loss/FRt2/detection_loss2.421859868146755 1loss/SRqqt2/detection_loss1.1414215680168465 trainer/global_step15649 _runtime6961.876865855 2loss/SRqqt2/assignment_loss0.9155540380684674 3loss/FBt1/assignment_loss0.0024222811341729766 1loss/SRqqt1/detection_loss1.1507652158821549 /loss/FBt2/detection_loss0.7689052274257973 4loss/FBt2/assignment_loss2.0266772338818008e-05 /loss/FRt1/detection_loss2.5727075431495905 epoch0 /loss/FBt1/detection_loss0.6629568068858456 &loss/total_loss14.771785836155269 $ _timestamp1.785802249053971e+09 _step1251߾& .loss/FRt2/detection_loss2.421859868146755 1loss/SRqqt2/detection_loss1.1414215680168465 trainer/global_step15649 _runtime6961.876865855 2loss/SRqqt2/assignment_loss0.9155540380684674 3loss/FBt1/assignment_loss0.0024222811341729766 1loss/SRqqt1/detection_loss1.1507652158821549 /loss/FBt2/detection_loss0.7689052274257973 4loss/FBt2/assignment_loss2.0266772338818008e-05 /loss/FRt1/detection_loss2.5727075431495905 epoch0 /loss/FBt1/detection_loss0.6629568068858456 &loss/total_loss14.771785836155269 $ _timestamp1.785802249053971e+09 _step1251 0loss/FRt1/assignment_loss2.2085043317393254 0loss/FRt2/assignment_loss2.1128393198237005 2loss/SRqqt1/assignment_loss0.8138293691102744 c66&(: ӚU gpu.0.gpu32.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp40.0 gpu.0.powerWatts109.732' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent36.577333333333335 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0x>\&(:  U) disk./.usagePercent32.34443912389351 disk.sda.out 188.90625* proc.memory.percent17.182016372680664 disk.drbd2041.in0 disk.drbd1550.in0 disk.rbd0.in0 disk.sda1.out0 disk.drbd1285.out0 disk.drbd1356.out0# disk.nvme0n1.out294.1474609375 disk.drbd1828.in0 disk.drbd1938.out0 disk.drbd1584.out0 disk.drbd1965.out0 cpu13.726826980796538 disk.sda2.in1.59375* proc.memory.availableMB38496.16015625 disk.drbd1584.in0 disk.drbd1965.in0 disk.drbd2041.out0 disk.sda1.in0 disk.drbd1226.in0 disk.sda2.out 188.90625 disk.drbd1817.out0 disk.drbd1552.in0 disk.drbd1828.out0 disk.dm-0.out294.1474609375 disk.drbd2078.in0 disk.drbd2078.out0 disk.drbd2111.in0 disk.drbd1817.in0 disk.drbd1289.in0 disk.drbd1938.in0 disk.drbd1356.in0 disk.drbd1931.in0 disk.drbd1552.out0 disk.drbd1285.in0 disk.drbd1550.out0 disk.drbd1289.out0 disk.drbd1226.out0 proc.cpu.threads29 disk.drbd1673.in0 network.sent 1.4420403e+07$ disk./.usageGB578.3514213562012 disk.rbd1.in0 disk.drbd1510.in0 disk.drbd1673.out0 disk.drbd1931.out0 disk.drbd2118.out0 disk.nvme0n1.in 4.64453125 disk.drbd2111.out0 disk.rbd0.out0 disk.sda.in1.59375! proc.memory.rssMB 11260.40625 disk.drbd2118.in0 disk.drbd1510.out0 disk.dm-0.in 4.64453125 network.recv790840 disk.rbd1.out28.03125$ memory_percent41.25952124595642HD$Y2&- _wandbruntime6983  _runtime6983g<& ' lr-AdamW/pg10.00040459755563282044 ' lr-AdamW/pg20.00040459755563282044 trainer/global_step15699 % _timestamp1.7858022710557082e+09 _runtime6983.878693599 _step1252՟& trainer/global_step15699 % _timestamp1.7858022710557082e+09 _runtime6983.878693599 _step1252 ' lr-AdamW/pg10.00040459755563282044 ' lr-AdamW/pg20.00040459755563282044 ʰh'2&- _wandbruntime6983  _runtime6983%'&O QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 trainer/global_step15699 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 270098612 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 106815588 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed61813295 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 126479872 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated14188287771136 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 % _timestamp1.7858022710566754e+09 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 104299679 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed276605824519404 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated283337192210432 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated269522721905664 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated284345870769664 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 270088954 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed283337087741440 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed13814382875136 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated283337192210432 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current249 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 _step1253 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 165789277 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14188161291264 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated61813297 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated45002542 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current251 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated269522721905664 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13814470304768 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13814382875136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated262825267140832 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated13780660221056 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 104299677 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated269522721905664 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated284345870769664 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 270088954 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed283337087741440 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed13814382875136 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated283337192210432 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current249 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 _step1253 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 165789277 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14188161291264 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated61813297 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated45002542 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current251 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated269522721905664 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13814470304768 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13814382875136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated262825267140832 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated13780660221056 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 104299677 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 J6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 165799800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated262842007313152 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated269539888668672 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed284363828149760 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 270115792 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 270106120 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated284363985817600 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed283355134124032 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _runtime6984.249038223 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 % _timestamp1.7858022714256601e+09 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 165799800 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current244 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed269539871629312 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13815349931520 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed276623442164946 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13815262494720 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14189062938112 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 104306322 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 106822609 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated270174790115328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 104306320 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated283355238600192 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed61817232 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 270106120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated283355238600192 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 104306320 trainer/global_step15699 _step1254 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed13815262494720 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed13781451891154 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 165809470 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 106822363 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed270174765211648 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated61817234 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 270115792 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated13781537694390 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 104306322 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 165809470 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated45005375 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed262841990273792`A&&M SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 165809470 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated45005375 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed262841990273792 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated269539888668672 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 165799800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated262842007313152 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated269539888668672 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed284363828149760 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 270115792 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 270106120 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated284363985817600 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed283355134124032 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _runtime6984.249038223 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 % _timestamp1.7858022714256601e+09 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 165799800 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current244 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed269539871629312 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13815349931520 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed276623442164946 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13815262494720 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14189062938112 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 104306322 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 106822609 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated270174790115328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 104306320 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated283355238600192 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed61817232 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 270106120 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated283355238600192 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 104306320 trainer/global_step15699 _step1254 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed13815262494720 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed13781451891154 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 165809470 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 106822363 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed270174765211648 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated61817234 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 270115792 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated13781537694390 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 104306322 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 jѤ2&- _wandbruntime6984  _runtime6984X & 0loss/FRt2/assignment_loss2.5357443815053897 epoch0 _step1255 3loss/FBt1/assignment_loss0.0007796504195416294 3loss/FBt2/assignment_loss2.454794171919056e-06 &loss/total_loss16.461386549298872 % _timestamp1.7858022714273496e+09 2loss/SRqqt1/assignment_loss0.9056535535653691 2loss/SRqqt2/assignment_loss0.7824619933843422 /loss/FRt2/detection_loss2.4198864708806194 .loss/FBt1/detection_loss1.358281800088783 /loss/FBt2/detection_loss1.4824100110714358 trainer/global_step15699 _runtime6984.250166907 .loss/FRt1/assignment_loss2.32682023100231 .loss/FRt1/detection_loss2.374761344945949 1loss/SRqqt1/detection_loss1.1277678310483072 1loss/SRqqt2/detection_loss1.1468168265926533FGf& 0loss/FRt2/assignment_loss2.5357443815053897 epoch0 _step1255 3loss/FBt1/assignment_loss0.0007796504195416294 3loss/FBt2/assignment_loss2.454794171919056e-06 &loss/total_loss16.461386549298872 % _timestamp1.7858022714273496e+09 2loss/SRqqt1/assignment_loss0.9056535535653691 2loss/SRqqt2/assignment_loss0.7824619933843422 /loss/FRt2/detection_loss2.4198864708806194 .loss/FBt1/detection_loss1.358281800088783 /loss/FBt2/detection_loss1.4824100110714358 trainer/global_step15699 _runtime6984.250166907 .loss/FRt1/assignment_loss2.32682023100231 .loss/FRt1/detection_loss2.374761344945949 1loss/SRqqt1/detection_loss1.1277678310483072 1loss/SRqqt2/detection_loss1.1468168265926533 e&(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp40.0 gpu.0.powerWatts110.586' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent36.862 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0eX&(:  U% memory_percent41.259193420410156 disk.drbd1552.in0 disk.drbd1550.out0 disk.drbd1289.in0 disk.sda1.in0 disk.drbd1817.out0 disk.drbd1584.in0! proc.memory.rssMB 11260.40625 network.recv792120 disk.nvme0n1.in 4.64453125 disk.drbd1356.in0 disk.drbd1673.in0 disk.drbd1285.in0 disk.drbd2078.out0 disk.drbd2111.in0 disk.rbd1.in0* disk./.usagePercent32.344442323961125 disk.drbd1931.in0 disk.drbd1931.out0 disk.drbd2041.out0 disk.drbd1510.in0 disk.dm-0.in 4.64453125 disk.drbd2118.out0 disk.drbd1828.out0 disk.drbd1289.out0 disk.rbd0.out0 disk.drbd1226.out0 disk.sda2.out 189.21875 disk.drbd1965.out0 disk.drbd2111.out0 network.sent 1.4465407e+07 disk.drbd1552.out0 disk.drbd1285.out0 disk.drbd1965.in0 disk.sda.in1.59375 disk.drbd1938.out0 disk.drbd1510.out0 disk.drbd1817.in0 cpu13.72643313660011$ disk./.usageGB578.3514785766602 disk.rbd0.in0 disk.drbd1584.out0# disk.nvme0n1.out294.3623046875 disk.drbd1550.in0 disk.rbd1.out28.03125 disk.drbd1673.out0 disk.drbd2041.in0 disk.drbd1226.in0 disk.drbd1356.out0 proc.cpu.threads29% proc.memory.availableMB 38496.375 disk.drbd1828.in0 disk.drbd2078.in0 disk.sda2.in1.59375 disk.sda.out 189.21875 disk.sda1.out0 disk.dm-0.out294.3623046875 disk.drbd1938.in0 disk.drbd2118.in0* proc.memory.percent17.182016372680664'^~sj&(:  U disk.drbd1226.in0 disk.sda2.out 189.48828125 disk.rbd0.out0 disk.drbd1226.out0 disk.drbd1828.in0 disk.drbd1938.out0 disk.sda1.in0 disk.drbd2111.out0 disk.drbd1673.in0 disk.drbd1510.in0 disk.drbd2118.in0 disk.drbd2111.in0 disk.sda.out 189.48828125 disk.drbd1584.in0 disk.rbd1.in0 network.sent 1.4510493e+07$ disk./.usageGB578.3514976501465 disk.sda1.out0 disk.drbd2041.in0% memory_percent41.258424520492554 disk.rbd0.in0 proc.cpu.threads29* disk./.usagePercent32.344443390650326 disk.drbd1550.out0 disk.rbd1.out 28.1572265625 disk.drbd1965.in0 disk.drbd1828.out0 disk.drbd1938.in0 disk.dm-0.in 4.64453125 disk.drbd1356.in0 disk.drbd1817.out0 disk.drbd1552.out0* proc.memory.availableMB38496.87890625 disk.drbd2078.out0 disk.drbd2118.out0* proc.memory.percent17.182016372680664 disk.drbd1285.in0 disk.nvme0n1.in 4.64453125 disk.drbd1550.in0 disk.drbd1510.out0 disk.drbd1356.out0 disk.drbd1817.in0 disk.drbd1552.in0 cpu13.72609425652556 network.recv793699 disk.drbd1289.in0 disk.sda.in1.59375$ disk.nvme0n1.out294.57080078125 disk.drbd2041.out0! disk.dm-0.out294.57080078125 disk.drbd2078.in0 disk.drbd1931.in0 disk.drbd1285.out0 disk.drbd1584.out0 disk.sda2.in1.59375 disk.drbd1965.out0 disk.drbd1673.out0 disk.drbd1931.out0! proc.memory.rssMB 11260.40625 disk.drbd1289.out0xl&(: ҦV gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp40.0 gpu.0.powerWatts109.976' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent36.65866666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0Uׅ2&- _wandbruntime7005  _runtime70057& trainer/global_step15749 % _timestamp1.7858022927750697e+09 _runtime7005.598038119 _step1256 ' lr-AdamW/pg10.00040588616495708575 ' lr-AdamW/pg20.00040588616495708575 OT& ' lr-AdamW/pg10.00040588616495708575 ' lr-AdamW/pg20.00040588616495708575 trainer/global_step15749 % _timestamp1.7858022927750697e+09 _runtime7005.598038119 _step1256 #ɽ2&- _wandbruntime7005  _runtime7005=Cp'&O [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed13824462881440 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current252 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13858379010560 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 166317279 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 166326935 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 270958764 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13858466440192 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 104631829 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14233357609984 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated270381060056064 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated13858466440192 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated14233490381312 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated62010147 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated13824548684564 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated285251429219840 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated270381060056064 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed285251271544832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed270381043016704 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current250 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 104631827 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed284239422027264 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated284239526496256 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 _runtime7005.602658192 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed45147174 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed284239422027264 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated277486824441396 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated263662275756832 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 270949106 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed270381043016704 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed271017913934848 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 trainer/global_step15749 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed263662258717472 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated45147424n1 &O LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 104631827 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed284239422027264 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated284239526496256 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 _runtime7005.602658192 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed45147174 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed284239422027264 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated277486824441396 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated263662275756832 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 270949106 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed270381043016704 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed271017913934848 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 trainer/global_step15749 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed263662258717472 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated45147424 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed13824462881440 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current252 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13858379010560 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 166317279 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 166326935 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 270958764 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13858466440192 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 104631829 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14233357609984 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated270381060056064 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated13858466440192 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated14233490381312 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated62010147 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated13824548684564 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated285251429219840 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated270381060056064 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed285251271544832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed270381043016704 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current250 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ,⣧2&- _wandbruntime7005  _runtime7005y&&M S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed284257468409856 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 107164343 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current261 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated13825426157898 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 107164080 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated277504442087050 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated270398226819072 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated14234391504896 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 126472704 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated62014084 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 166337472 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed270398209779712 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 270975944 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 104638472 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime7005.971726237 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13859346066944 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated284257572886016 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14234265032192 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 270966272 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 104638470 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed285269386083840 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed271035121051648 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated263679015929152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated45150259 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed263678998889792 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current263 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 270975944 trainer/global_step15749 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13859258630144 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 104638470 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed62014082 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated271035145955328 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed45149998 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 151376384 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 166327802 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 104638472 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated284257572886016 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121C?k&&M ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed45149998 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 151376384 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 166327802 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 104638472 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated284257572886016 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed284257468409856 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 107164343 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current261 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated13825426157898 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 107164080 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated277504442087050 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated270398226819072 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated14234391504896 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 126472704 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated62014084 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 166337472 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed270398209779712 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 270975944 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 104638472 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime7005.971726237 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13859346066944 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated284257572886016 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14234265032192 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 270966272 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 104638470 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed285269386083840 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed271035121051648 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated263679015929152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated45150259 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed263678998889792 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current263 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 270975944 trainer/global_step15749 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13859258630144 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 104638470 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed62014082 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated271035145955328 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 ベ2'- _wandbruntime7005  _runtime7005?' 0loss/FRt1/assignment_loss2.2499275021255016 0loss/FRt2/assignment_loss2.1157519665029314 2loss/SRqqt1/assignment_loss0.6046319648635406 3loss/FBt1/assignment_loss5.639980737992545e-06 3loss/FBt2/assignment_loss5.188305207766353e-06 1loss/SRqqt1/detection_loss1.0607346242460876 &loss/total_loss15.206904174499178 # _timestamp1.78580229315003e+09 .loss/FRt2/detection_loss2.237910565816694 1loss/SRqqt2/detection_loss1.0210816081996787 .loss/FBt2/detection_loss1.059707901329689 trainer/global_step15749 _runtime7005.972854167 _step1259 epoch0 1loss/SRqqt2/assignment_loss0.808925957802464 .loss/FRt1/detection_loss2.488306790031493 /loss/FBt1/detection_loss1.5599144652951509b ' 1loss/SRqqt2/assignment_loss0.808925957802464 .loss/FRt1/detection_loss2.488306790031493 /loss/FBt1/detection_loss1.5599144652951509 0loss/FRt1/assignment_loss2.2499275021255016 0loss/FRt2/assignment_loss2.1157519665029314 2loss/SRqqt1/assignment_loss0.6046319648635406 3loss/FBt1/assignment_loss5.639980737992545e-06 3loss/FBt2/assignment_loss5.188305207766353e-06 1loss/SRqqt1/detection_loss1.0607346242460876 &loss/total_loss15.206904174499178 # _timestamp1.78580229315003e+09 .loss/FRt2/detection_loss2.237910565816694 1loss/SRqqt2/detection_loss1.0210816081996787 .loss/FBt2/detection_loss1.059707901329689 trainer/global_step15749 _runtime7005.972854167 _step1259 epoch0 _S'(: U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp40.0 gpu.0.powerWatts132.308' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent44.102666666666664 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0ug'(:  U$ disk.nvme0n1.out294.83251953125 disk.drbd1938.out0 disk.sda2.out 190.16796875 disk.drbd1817.in0 disk.drbd1584.out0 disk.drbd1552.in0 disk.drbd1552.out0 disk.drbd1828.in0 disk.drbd1931.out0 disk.drbd2041.in0 disk.drbd1938.in0* disk./.usagePercent32.344442323961125 disk.drbd1226.in0 disk.drbd1828.out0 disk.drbd1817.out0 disk.drbd1510.in0' proc.memory.availableMB 38496.09375! proc.memory.rssMB 11260.40625 disk.drbd1584.in0 disk.drbd1289.out0$ memory_percent41.25962257385254 network.sent 1.4513532e+07 disk.drbd1673.in0 disk.drbd1285.in0 network.recv794621 disk.dm-0.in 4.64453125 disk.drbd1356.out0 disk.drbd1673.out0 disk.sda1.in0 disk.sda2.in1.59375! disk.dm-0.out294.83251953125 disk.drbd1226.out0 disk.drbd2078.in0 disk.drbd2118.in0 disk.drbd2118.out0 disk.drbd1965.in0 disk.drbd1289.in0 disk.drbd1931.in0 disk.drbd1550.in0 cpu13.725791871300002 proc.cpu.threads29 disk.rbd1.out 28.3369140625 disk.drbd2111.out0 disk.drbd2111.in0 disk.drbd2041.out0 disk.drbd1550.out0 disk.sda.out 190.16796875* proc.memory.percent17.182016372680664 disk.drbd1356.in0$ disk./.usageGB578.3514785766602 disk.rbd0.in0 disk.drbd1285.out0 disk.drbd2078.out0 disk.rbd1.in0 disk.rbd0.out0 disk.sda.in1.59375 disk.sda1.out0 disk.drbd1510.out0 disk.drbd1965.out0 disk.nvme0n1.in 4.64453125@2'- _wandbruntime7027  _runtime7027&3' % lr-AdamW/pg10.000407174774281351 % lr-AdamW/pg20.000407174774281351 trainer/global_step15799 % _timestamp1.7858023150112767e+09 _runtime7027.834257262 _step1260Mߞ' % _timestamp1.7858023150112767e+09 _runtime7027.834257262 _step1260 % lr-AdamW/pg10.000407174774281351 % lr-AdamW/pg20.000407174774281351 trainer/global_step15799 V2'- _wandbruntime7027  _runtime7027<L'O YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 271818916 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 104963979 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated14278639905280 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14278509231104 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed13868351344948 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated]<#0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 107498390 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 % _timestamp1.7858023150122294e+09 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated285141860782080 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 166854937 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 107498140 _runtime7027.835439087 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed286156779005952 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed271878269774848 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed264499267333472 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _step1261 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13902462575616 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 104963977 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 166845281 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 104963977 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current250 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 166845281 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed278367618678420 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed285141756313088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated45291393 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 271809258 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated271239398206464 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current248 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated271878294678528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated278367721520904 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated271239398206464 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated62206997 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 trainer/global_step15799 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed45291145 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed285141756313088 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 1668549371 ''O N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13902462575616 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 104963977 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 166845281 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 104963977 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current250 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 166845281 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed278367618678420 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed285141756313088 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated45291393 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 271809258 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated271239398206464 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current248 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated271878294678528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated278367721520904 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated271239398206464 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated62206997 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 trainer/global_step15799 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed45291145 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed285141756313088 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 166854937 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 271818916 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 104963979 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated14278639905280 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14278509231104 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed13868351344948 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 107498390 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 % _timestamp1.7858023150122294e+09 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated285141860782080 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 166854937 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 107498140 _runtime7027.835439087 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed286156779005952 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed271878269774848 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed264499267333472 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _step1261 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 J6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 104970622 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 166855804 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current243 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated278385339166558 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed45293982 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed271256547930112 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13903254765568 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated14279542355968 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 _runtime7028.210705212 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated271895501795328 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated271256564969472 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed285159802695680 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 271826424 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 trainer/global_step15799 _step1262 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated285159907171840 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 166865474 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 271836096 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13903342202368 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated13869314621406 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 271826424 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 104970620 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 104970620 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 107505157 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed13869228818170 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed271256547930112 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed13903254765568 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated271256564969472 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14279409591808 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed264516007505792 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13903342202368 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated45294223 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current241 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed62210932 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 271836096 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 107504914 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed286174886483456 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated264516024545152 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 104970622 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 166855804 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current243 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated278385339166558 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed45293982 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed271256547930112 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13903254765568 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated14279542355968 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 _runtime7028.210705212 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated271895501795328 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated271256564969472 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed285159802695680 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 271826424 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 trainer/global_step15799 _step1262 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated285159907171840 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 166865474 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 271836096 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13903342202368 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated13869314621406 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 271826424 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 104970620 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 104970620 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 107505157 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed13869228818170 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed271256547930112 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed13903254765568 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated271256564969472 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14279409591808 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed264516007505792 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13903342202368 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated45294223 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current241 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed62210932 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 271836096 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 107504914 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed286174886483456 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated264516024545152 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 EDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed286044090598912 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 167382939 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated14323841473024 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed45433250 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed287062332219392 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13946371281408 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 107837346 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed13946371281408 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed13912239808456 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 trainer/global_step15849 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed272738625614848 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 167382939 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated286044195067904 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated13946458711040 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated62403847 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14323706604544 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed279248515757928 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _runtime7049.798559408 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed62403845 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed272097719317504 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed286044090598912 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 107837095 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 167373283 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 % _timestamp1.7858023369752705e+09 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated45433499 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated279248618600412 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated265336292988832 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 105296127 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated272097736356864 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 107837095 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 167373283 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 % _timestamp1.7858023369752705e+09 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated45433499 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated279248618600412 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated265336292988832 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 105296127 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated272097736356864 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed286044090598912 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 167382939 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated14323841473024 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed45433250 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed287062332219392 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13946371281408 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 107837346 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed13946371281408 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed13912239808456 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 trainer/global_step15849 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 w{'(: щU gpu.0.gpu24.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp41.0 gpu.0.powerWatts130.083' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent43.361 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0%jZ;'(:  ȁU disk.drbd1931.out0 disk.drbd1817.out0 disk.rbd1.out 28.3994140625 cpu13.727141834791 disk.drbd1828.in0 disk.drbd1550.in0 disk.sda.in1.59375 disk.sda.out 190.83984375 disk.drbd1510.in0 disk.drbd1931.in0 disk.sda1.in0 disk.drbd1938.in0 disk.drbd1584.out0 disk.nvme0n1.in 4.64453125 disk.sda2.in1.59375 disk.drbd1965.in0 disk.drbd1673.in0 disk.drbd1552.out0 disk.drbd1550.out0 disk.drbd1938.out0 disk.drbd2111.out0$ disk./.usageGB578.3515014648438* proc.memory.percent17.182016372680664 network.recv797835 disk.drbd1289.in0 disk.drbd1285.in0 disk.drbd1226.out0 disk.drbd1817.in0 disk.rbd0.in0 disk.drbd1226.in0 disk.sda1.out0 disk.rbd1.in0) proc.memory.availableMB 38495.5390625 disk.drbd2078.out0 disk.drbd2118.out0$ disk.nvme0n1.out295.32373046875 disk.drbd2041.out0 disk.drbd1828.out0 proc.cpu.threads29 disk.drbd1673.out0  ]*disk.drbd1285.out0 disk.drbd2078.in0 disk.drbd1356.in0 disk.sda2.out 190.83984375$ memory_percent41.26046895980835 disk.drbd2118.in0! proc.memory.rssMB 11260.40625 network.sent 1.4603768e+07 disk.drbd1289.out0 disk.drbd1510.out0 disk.dm-0.in 4.64453125) disk./.usagePercent32.34444360398817! disk.dm-0.out295.32373046875 disk.drbd2111.in0 disk.drbd1965.out0 disk.drbd2041.in0 disk.drbd1552.in0 disk.rbd0.out0 disk.drbd1584.in0 disk.drbd1356.out0drA2'- _wandbruntime7050  _runtime7050 )&'M E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed13947250900992 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed279266133403470 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated13913203084914 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 167393476 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 167383806 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated272114903119872 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 167393476 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed287080437517824 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 _runtime 7050.16370341 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 105302770 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated287080603574272 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 272686576 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 105302772 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current270 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated45436327 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated272114903119872 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 166056448 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed265353016121792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 167383806 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 272696248 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated286062241457664 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed286062136981504 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13947338337792 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 105302770 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 _step1266 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated272755857635328 trainer/global_step15849 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13947338337792 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13947250900992 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 272686576 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 107843839 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated14324745938944 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 141152768 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14324604786176 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed272755832731648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 107843839 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated14324745938944 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 141152768 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14324604786176 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed272755832731648 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed279266133403470 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated13913203084914 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 167393476 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 167383806 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated272114903119872 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 167393476 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed287080437517824 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 _runtime 7050.16370341 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 105302770 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated287080603574272 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 272686576 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 105302772 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current270 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated45436327 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated272114903119872 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 166056448 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed265353016121792 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 167383806 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 272696248 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated286062241457664 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed286062136981504 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13947338337792 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 105302770 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 _step1266 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated272755857635328 trainer/global_step15849 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13947338337792 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed13947250900992 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 272686576 EDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 167910941 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current261 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed45576449 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 273529562 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated14368997196288 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated62600697 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 167901285 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated45576708 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated287968003554816 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated272956074507264 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated273599006358528 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated280129515679920 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed272956057467904 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 167910941 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13990367416832 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed62600695 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current259 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 105628277 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 273539220 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed287967845879808 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated13956214075088 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13990454846464 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed286946424884736 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 105628279 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed273598981454848 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed266173284565472 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 108177144 trainer/global_step15899 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 108177405 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0!G3''O QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 105628277 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 _step1269 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated13990454846464 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 167910941 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current261 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed45576449 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 273529562 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated14368997196288 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated62600697 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 167901285 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated45576708 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated287968003554816 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated272956074507264 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated273599006358528 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated280129515679920 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed272956057467904 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 167910941 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed13990367416832 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed62600695 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current259 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 105628277 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 273539220 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed287967845879808 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated13956214075088 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated13990454846464 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed286946424884736 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 105628279 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed273598981454848 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed266173284565472 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 108177144 trainer/global_step15899 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 108177405 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed13990367416832 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated266173301604832 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 J6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated13957091548422 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 167911808 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _step1270 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 167921478 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current249 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 273556400 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13991334473216 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 % _timestamp1.7858023592666955e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 108183920 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed266190024737792 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated286964575743488 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed272973224230912 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13991334473216 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 105634922 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 167921478 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated287986113499136 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14369769356800 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 105634922 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated272973241270272 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed45579288 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 _runtime7072.090040464 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 105634920 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated14369900023808 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated266190041777152 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed286964471267328 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 273556400 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 108184171E&'M OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 105634922 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated272973241270272 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed45579288 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 _runtime7072.090040464 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 105634920 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated14369900023808 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated266190041777152 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed286964471267328 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 273556400 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 108184171 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated13957091548422 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 167911808 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _step1270 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 167921478 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current249 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 273556400 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated13991334473216 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 % _timestamp1.7858023592666955e+09 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 108183920 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed266190024737792 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated286964575743488 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed272973224230912 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated13991334473216 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 105634922 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 167921478 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated287986113499136 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14369769356800 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 2'- _wandbruntime7072  _runtime7072)' /loss/FRt2/assignment_loss2.158918563355791 4loss/FBt1/assignment_loss3.2511621193200634e-08 4loss/FBt2/assignment_loss1.9072904596918584e-06 epoch0 /loss/FRt1/detection_loss2O>.6333254368321315 /loss/FBt2/detection_loss0.6788269999411164 %loss/total_loss16.10508225784052 _step1271 1loss/SRqqt2/detection_loss1.0700586883848984 /loss/FBt1/detection_loss1.6763293008235367 trainer/global_step15899 0loss/FRt1/assignment_loss2.4788234725191787 2loss/SRqqt1/assignment_loss1.0387640066204409 2loss/SRqqt2/assignment_loss0.9457148643238631 /loss/FRt2/detection_loss2.2098100746961986 1loss/SRqqt1/detection_loss1.2145089105412852 % _timestamp1.7858023592684026e+09 _runtime7072.091242241$:' 4loss/FBt1/assignment_loss3.2511621193200634e-08 4loss/FBt2/assignment_loss1.9072904596918584e-06 epoch0 /loss/FRt1/detection_loss2.6333254368321315 /loss/FBt2/detection_loss0.6788269999411164 %loss/total_loss16.10508225784052 _step1271 1loss/SRqqt2/detection_loss1.0700586883848984 /loss/FBt1/detection_loss1.6763293008235367 trainer/global_step15899 0loss/FRt1/assignment_loss2.4788234725191787 2loss/SRqqt1/assignment_loss1.0387640066204409 2loss/SRqqt2/assignment_loss0.9457148643238631 /loss/FRt2/detection_loss2.2098100746961986 1loss/SRqqt1/detection_loss1.2145089105412852 % _timestamp1.7858023592684026e+09 _runtime7072.091242241 /loss/FRt2/assignment_loss2.158918563355791 G氜'(: U gpu.0.gpu24.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp41.0 gpu.0.powerWatts121.828' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent40.60933333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0E=Va'(:  U disk.drbd2041.out0 disk.drbd1828.out0 network.sent 1.4672904e+07 disk.drbd1552.out0 disk.sda.in1.59375 disk.drbd2118.out0 disk.drbd1673.in0 disk.drbd1931.out0$ memory_percent41.25956892967224 disk.drbd1289.in0 disk.rbd0.in0 disk.drbd1510.in0 disk.drbd1931.in0 cpu13.728723442355312 disk.rbd1.in0 proc.cpu.threads29$ disk./.usageGB578.3528861999512 disk.drbd2078.in0 disk.drbd1817.out0 disk.drbd1285.out0 disk.drbd1289.out0 disk.sda1.in0 disk.drbd1226.in0 disk.sda2.out 191.69921875 disk.rbd1.out 28.525390625 disk.drbd1965.out0 disk.drbd2041.in0 disk.sda.out 191.69921875 disk.drbd1817.in0 network.recv800927 disk.drbd1285.in0 disk.drbd2078.out0 disk.drbd2111.in0 disk.drbd2111.out0( disk./.usagePercent32.3445210456244 disk.drbd1965.in0 disk.drbd1828.in0 disk.sda1.out0 disk.drbd1510.out0 disk.drbd2118.in0! disk.nvme0n1.out 295.95703125* proc.memory.availableMB38496.12890625 disk.rbd0.out0 disk.dm-0.in 4.64453125 disk.drbd1552.in0 disk.drbd1550.out0 disk.dm-0.out 295.95703125 disk.drbd1226.out0 disk.drbd1584.out0 disk.nvme0n1.in 4.64453125 disk.drbd1673.out0! proc.memory.rssMB 11260.40625 disk.drbd1938.in0 disk.drbd1938.out0 disk.drbd1584.in0 disk.drbd1356.in0 disk.drbd1356.out0* proc.memory.percent17.182016372680664 disk.drbd1550.in0 disk.sda2.in1.59375'2'- _wandbruntime7094  _runtime7094p' % lr-AdamW/pg10.000411040602254147 % lr-AdamW/pg20.000411040602254147 trainer/global_step15949 $ _timestamp1.785802381303451e+09 _runtime7094.126423722 _step1272Yx_' % lr-AdamW/pg10.000411040602254147 % lr-AdamW/pg20.000411040602254147 trainer/global_step15949 $ _timestamp1.785802381303451e+09 _runtime7094.126423722 _step1272 K2'- _wandbruntime7094  _runtime7094P''O A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated62797547 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 108517101 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated267010310220832 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed267010293181472 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 _step1273 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14034450981888 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed62797545 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed45719556 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14000016735472 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated273814412657664 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14414032619520 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated14414165390848 % _timestamp1.7858023813044484e+09 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 274399372 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed273814395618304 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed274459337294848 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed281010309916944 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated287848863639552 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 105960427 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 168438943 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14000102538596 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed273814395618304 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 168438943 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 105960427 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated273814412657664 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current257 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed287848759170560 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated287848863639552 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current259 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed288873369914368 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14034363552256 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated274459362198528 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated281010412759428~]/''O C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14034450981888 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 105960429 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 trainer/global_step15949 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed287848759170560 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 105960427 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 168438943 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14000102538596 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed273814395618304 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 168438943 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 105960427 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated273814412657664 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current257 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed287848759170560 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated287848863639552 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current259 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed288873369914368 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14034363552256 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated274459362198528 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated281010412759428 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated62797547 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 108517101 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated267010310220832 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed267010293181472 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 _step1273 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14034450981888 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed62797545 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed45719556 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14000016735472 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated273814412657664 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14414032619520 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated14414165390848 % _timestamp1.7858023813044484e+09 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 274399372 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed273814395618304 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed274459337294848 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed281010309916944 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated287848863639552 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 108524146 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 108523879 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated62801484 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14000980011930 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed287866805553152 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated14035330608640 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated14415068677120 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated267027050393152 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated274476569315328 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated281028030405082 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 105967070 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed273831562381312 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14035243171840 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current265 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 168449480 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed273831562381312 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed62801482 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 274416552 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14414938010112 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed267027033353792 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14000894208694 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed45722397 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed288891482421760 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed281027927562486 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 168439810 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 UGDeviceStatsMonitor.on_train_batch+(_end/reserved_bytes.small_pool.current 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated288891637992448 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 105967070 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 168449480 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated45722662 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated273831579420672 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 274416552 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current267 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 $ _timestamp1.785802381675067e+09 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 274406880 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed14035243171840 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 168439810 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed274476544411648 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 274416552 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current267 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 $ _timestamp1.785802381675067e+09 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 274406880 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed14035243171840 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 168439810 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed274476544411648 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 108524146 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 108523879 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated62801484 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14000980011930 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed287866805553152 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated14035330608640 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated14415068677120 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated267027050393152 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated274476569315328 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated281028030405082 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 105967070 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed273831562381312 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14035243171840 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current265 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 168449480 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed273831562381312 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed62801482 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 274416552 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14414938010112 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed267027033353792 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14000894208694 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 M2'- _wandbruntime7094  _runtime7094 ' % _timestamp1.7858023816770349e+09 2loss/FBt1/assignment_loss0.024345275759492323 -loss/FBt1/detection_loss1.37935136500746 trainer/global_step15949 2loss/SRqqt2/assignment_loss0.9225180703107008 4loss/FBt2/assignment_loss1.7185644991535298e-05 .loss/FRt2/detection_loss2.283872845729715 _runtime7094.499871709 0loss/FRt2/assignment_loss2.2915215484405818 .loss/FRt1/detection_loss2.520719336345792 &loss/total_loss15.511407518552401 epoch0 _step1275 0loss/FRt1/assignment_loss2.5530915267088194 1loss/SRqqt1/assignment_loss0.743308824260327 1loss/SRqqt1/detection_loss0.9288351816419316 1loss/SRqqt2/detection_loss1.0801307516057015 /loss/FBt2/detection_loss0.7836956070968881D4c2' 1loss/SRqqt2/detection_loss1.0801307516057015 /loss/FBt2/detection_loss0.7836956070968881 % _timestamp1.7858023816770349e+09 2loss/FBt1/assignment_loss0.024345275759492323 -loss/FBt1/detection_loss1.37935136500746 trainer/global_step15949 2loss/SRqqt2/assignment_loss0.9225180703107008 4loss/FBt2/assignment_loss1.7185644991535298e-05 .loss/FRt2/detection_loss2.283872845729715 _runtime7094.499871709 0loss/FRt2/assignment_loss2.2915215484405818 .loss/FRt1/detection_loss2.520719336345792 &loss/total_loss15.511407518552401 epoch0 _step1275 0loss/FRt1/assignment_loss2.5530915267088194 1loss/SRqqt1/assignment_loss0.743308824260327 1loss/SRqqt1/detection_loss0.9288351816419316 |ib'(:  U disk.drbd1817.out0 disk.drbd1931.in0 disk.drbd1828.in0 disk.drbd1938.out0 disk.drbd1965.in0$ memory_percent41.25921130180359 disk.drbd2041.out0 disk.drbd1510.in0 disk.drbd1356.out0 disk.rbd1.out 28.587890625 disk.drbd1550.out0 disk.drbd1356.in0 disk.sda.out 192.07421875 disk.dm-0.out 296.08984375 disk.nvme0n1.in 4.64453125 disk.drbd1285.out0 disk.drbd2118.in0 disk.drbd1289.in0 disk.drbd1226.out0 disk.drbd2078.out0 disk.drbd1584.in0 disk.drbd2111.out0 disk.rbd0.in0 disk.drbd2111.in0 disk.drbd1965.out0 disk.drbd1673.out0 disk.drbd1931.out0 disk.drbd2041.in0 network.sent 1.4718078e+07! proc.memory.rssMB 11260.40625* proc.memory.percent17.182016372680664 disk.drbd1938.in0 disk.drbd1817.in0 disk.rbd1.in0 disk.drbd1552.out0 disk.rbd0.out0 disk.drbd2078.in0 disk.drbd2118.out0 disk.sda2.out 192.07421875 network.recv802572 disk.drbd1550.in0 disk.drbd1510.out0 disk.drbd1226.in0 disk.sda2.in1.59375 proc.cpu.threads29 disk.drbd1289.out0 disk.sda1.out0 disk.drbd1285.in0 disk.dm-0.in 4.64453125 disk.drbd1584.out0* proc.memory.availableMB38496.36328125 disk.drbd1673.in0 disk.drbd1828.out0 disk.sda.in1.59375 disk.sda1.in0 cpu13.729519099374933) disk./.usagePercent32.34451784555679$ disk./.usageGB578.3528289794922 disk.drbd1552.in0! disk.nvme0n1.out 296.08984375ez'(: ™V gpu.0.gpu38.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp41.0 gpu.0.powerWatts101.456' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent33.81866666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors02{'(: U gpu.0.gpu24.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp41.0 gpu.0.powerWatts118.515' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent39.505 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0qܝZe'(:  U disk.drbd1931.in0* proc.memory.percent17.182016372680664$ disk./.usageGB578.3528861999512 disk.drbd1285.out0 disk.drbd2118.out0 disk.drbd1965.in0 disk.drbd1938.in0 cpu13.73036364620771 disk.sda1.out0 disk.drbd1285.in0 disk.dm-0.in 4.64453125 disk.drbd2078.in0 disk.drbd1584.out0 disk.drbd2118.in0 disk.sda2.in1.59375% memory_percent41.260021924972534 disk.drbd1673.out0 disk.drbd2041.in0 disk.drbd1552.in0! proc.memory.rssMB 11260.40625 disk.drbd1552.out0 disk.drbd1356.out0 disk.rbd1.in0 disk.drbd1673.in0 disk.drbd1828.in0 disk.rbd0.in0 disk.sda.in1.59375 disk.sda.out 192.38671875 disk.drbd1510.out0 disk.drbd1965.out0 disk.drbd1931.out0 disk.drbd1226.in0# disk.nvme0n1.out296.5732421875 disk.drbd1289.in0 proc.cpu.threads29 disk.drbd1510.in0 disk.drbd1226.out0 disk.drbd1356.in0* proc.memory.availableMB38495.83203125 disk.drbd1828.out0 disk.drbd1550.in0 disk.drbd1289.out0 disk.sda1.in0 disk.drbd1817.in0 disk.drbd1817.out0 network.sent 1.4763068e+07 disk.rbd1.out 28.587890625 disk.rbd0.out0 disk.sda2.out 192.38671875 disk.dm-0.out296.5732421875 network.recv803935( disk./.usagePercent32.3445210456244 disk.nvme0n1.in 4.64453125 disk.drbd2111.out0 disk.drbd2041.out0 disk.drbd1550.out0 disk.drbd1938.out0 disk.drbd2078.out0 disk.drbd1584.in0 disk.drbd2111.in0 2'- _wandbruntime7115  _runtime7115t:' ' lr-AdamW/pg10.00041232921157841225 ' lr-AdamW/pg20.00041232921157841225 trainer/global_step15999 % _timestamp1.7858024030111005e+09 _runtime7115.834072255 _step1276-n' _runtime7115.834072255 _step1276 ' lr-AdamW/pg10.00041232921157841225 ' lr-AdamW/pg20.00041232921157841225 trainer/global_step15999 % _timestamp1.7858024030111005e+09 7V2'- _wandbruntime7115  _runtime7115Rb''O RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 108857309 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 275259524 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14459215622144 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 _step1277 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated274672750808064 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated267847318836832 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 trainer/global_step15999 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 _runtime7115.838657095 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 106292579 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 168966945 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current256 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed289778908756992 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 275249866 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated275319718038528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14078447117312 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated45862912 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14078359687680 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 106292577 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed288751093456384 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated62994397 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated281891309838936 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed267847301797472 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14043991002104 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated288751197925376 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 % _timestamp1.7858024030120482e+09 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 126479872 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14043905198980 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated14459342102016 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed281891206996452 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed275319693134848 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 108857053 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current254 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated289779060140544 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 168966945 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed288751093456384 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121d<'O VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 % _timestamp1.7858024030120482e+09 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 126479872 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14043905198980 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated14459342102016 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed281891206996452 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed275319693134848 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 108857053 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current254 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated289779060140544 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 168966945 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed288751093456384 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.currentsW,0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 108857309 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 275259524 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14459215622144 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 _step1277 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated274672750808064 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated267847318836832 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 trainer/global_step15999 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 _runtime7115.838657095 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 106292579 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 168966945 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current256 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed289778908756992 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 275249866 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated275319718038528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14078447117312 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated45862912 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14078359687680 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 106292577 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed288751093456384 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated62994397 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated281891309838936 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed267847301797472 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14043991002104 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated288751197925376 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 *2'- _wandbruntime7116  _runtime7116&'M K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 168967812 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 275267032 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14460115794432 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated267864059009152 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed275336900251648 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed281908824641994 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed288769139838976 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 108864079 % _timestamp1.7858024033842938e+09 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 108863834 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 106299220 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated289797175811072 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 275276704 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed267864041969792 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated14079326744064 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed14079239307264 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 _step1278 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 168977482 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 275267032 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated274689917571072 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 168967812 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 108863834 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 106299220 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated289797175811072 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 275276704 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed267864041969792 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated14079326744064 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed14079239307264 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 _step1278 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 168977482 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 275267032 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated274689917571072 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 168967812 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed288769139838976 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 108864079 % _timestamp1.7858024033842938e+09 BT2'- _wandbruntime7116  _runtime7116`R ' 0loss/FRt1/assignment_loss2.2805020823198205 0loss/FRt2/assignment_loss2.6009044418596243 2loss/SRqqt1/assignment_loss0.9729321155831064 /loss/FRt2/detection_loss2.6626297043610925 0loss/SRqqt1/detection_loss1.146152243321149 0loss/SRqqt2/detection_loss1.140805766093586 trainer/global_step15999 _runtime7116.209473373 2loss/SRqqt2/assignment_loss0.8061334785894054 epoch0 _step1279 4loss/FBt1/assignment_loss2.2351738238057806e-08 4loss/FBt2/assignment_loss1.2181531512389654e-06 .loss/FRt1/detection_loss2.449659732597716 /loss/FBt1/detection_loss1.3546540145762265 .loss/FBt2/detection_loss1.358091092435643 %loss/total_loss16.77246591224226 % _timestamp1.7858024033863173e+09HY' epoch0 _step1279 4loss/FBt1/assignment_loss2.2351738238057806e-08 4loss/FBt2/assignment_loss1.2181531512389654e-06 .loss/FRt1/detection_loss2.449659732597716 /loss/FBt1/detection_loss1.3546540145762265 .loss/FBt2/detection_loss1.358091092435643 %loss/total_loss16.77246591224226 % _timestamp1.7858024033863173e+09 0loss/FRt1/assignment_loss2.2805020823198205 0loss/FRt2/assignment_loss2.6009044418596243 2loss/SRqqt1/assignment_loss0.9729321155831064 /loss/FRt2/detection_loss2.6626297043610925 0loss/SRqqt1/detection_loss1.146152243321149 0loss/SRqqt2/detection_loss1.140805766093586 trainer/global_step15999 _runtime7116.209473373 2loss/SRqqt2/assignment_loss0.8061334785894054 t<'(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp41.0 gpu.0.powerWatts128.064' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent42.687999999999995 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0c'(:  U disk.drbd1289.in0( disk./.usagePercent32.3445221123136 disk.sda1.out0 disk.drbd1285.out0 disk.drbd2118.out0 disk.drbd1965.in0 disk.drbd1965.out0 disk.drbd2041.in0 disk.drbd1289.out0 disk.drbd1938.out0 disk.drbd1226.out0 disk.drbd1584.out0 disk.drbd1356.in0 disk.drbd1673.in0 disk.sda.out 192.94921875 disk.sda.in1.59375 disk.drbd2111.in0 disk.drbd2118.in0 disk.dm-0.in 4.64453125 disk.drbd2111.out0 disk.drbd1552.in0 disk.drbd2078.in0 disk.rbd1.out 28.6201171875 disk.drbd1550.out0* proc.memory.percent17.182016372680664 disk.drbd1510.out0 disk.drbd1226.in0 disk.drbd1584.in0 disk.nvme0n1.in 4.64453125 disk.drbd1817.in0 disk.rbd1.in0 disk.drbd1828.out0( proc.memory.availableMB 38496.078125# disk.nvme0n1.out296.7373046875 cpu13.73007352615899 network.recv805089 disk.drbd1817.out0 disk.drbd2041.out0 disk.drbd1828.in0 disk.drbd1550.in0! proc.memory.rssMB 11260.40625 disk.sda2.out 192.94921875 disk.rbd0.out0$ memory_percent41.25964641571045 proc.cpu.threads29 disk.dm-0.out296.7373046875 disk.drbd1938.in0 disk.drbd1673.out0 disk.drbd1552.out0 disk.rbd0.in0 network.sent 1.4766331e+07 disk.drbd1356.out0 disk.sda2.in1.59375 disk.drbd1931.in0 disk.drbd1931.out0$ disk./.usageGB578.3529052734375 disk.sda1.in0 disk.drbd1285.in0 disk.drbd1510.in0 disk.drbd2078.out0[2'- _wandbruntime7138  _runtime7138S ' ' lr-AdamW/pg10.00041361782090267756 ' lr-AdamW/pg20.00041361782090267756 trainer/global_step16049 % _timestamp1.7858024254601953e+09 _runtime7138.283145193 _step1280!' ' lr-AdamW/pg20.00041361782090267756 trainer/global_step16049 % _timestamp1.7858024254601953e+09 _runtime7138.283145193 _step1280 ' lr-AdamW/pg10.00041361782090267756 iE2'- _wandbruntime7138  _runtime7138/P 'O NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 106624729 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed289653427742208 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated275531088958464 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 106624727 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14087793662488 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current241 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 106624729 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated14504527635456 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14122443252736 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14122355823104 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated290684601513984 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 169494947 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed63191245 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 _runtime 7138.28431187 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 _step1281 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current243 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated289653532211200 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887N}''O QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 106624729 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed289653427742208 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated275531088958464 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 106624727 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14087793662488 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current241 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 106624729 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated14504527635456 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14122443252736 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14122355823104 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated290684601513984 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 169494947 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed63191245 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 _runtime 7138.28431187 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 _step1281 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current243 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated289653532211200 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 x_2'- _wandbruntime7138  _runtime7138(:&'M LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 106631370 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated63195184 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 276136856 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 109203466 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 106631372 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 169505484 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed46008284 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed289671474124800 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime7138.673982447 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14123235442688 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated290702714931712 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14088756938946 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 169505484 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated276197280995328 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated282789824564098 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 106631370 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed275548238682112 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 276127184 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14088671135710 trainer/global_step16049 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed290702559361024 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed282789721721502 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed63195182 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed276197256091648 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated14123322879488 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current244 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 276136856 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed14123235442688 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated289671578600960 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated14505433936384 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current242 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 $ _timestamp1.785802425850492e+09 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated275548255721472 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 109203710 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 _step1282 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 276127184 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed63195182 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed276197256091648 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated14123322879488 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current244 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 276136856 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed14123235442688 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated289671578600960 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated14505433936384 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current242 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 $ _timestamp1.785802425850492e+09 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated275548255721472 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 109203710 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 _step1282 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 276127184 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 109203466 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 106631372 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 169505484 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed46008284 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed289671474124800 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime7138.673982447 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14123235442688 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated290702714931712 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14088756938946 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 169505484 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated276197280995328 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated282789824564098 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 106631370 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed275548238682112 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 !2'- _wandbruntime7138  _runtime71385u ' _runtime7138.675780272 _step1283 0loss/FRt2/assignment_loss1.9860829311081125 2loss/SRqqt1/assignment_loss0.8011794430237975 2loss/SRqqt2/assignment_loss0.8403404908551072 3loss/FBt2/assignment_loss1.922730201576703e-08 /loss/FRt1/detection_loss2.3394451899373014 /loss/FRt1/assignment_loss2.667590119268583 %loss/total_loss14.93563771550685 trainer/global_step16049 1loss/SRqqt1/detection_loss0.8755761191440636 /loss/FBt1/detection_loss1.3043279108177457 epoch0 1loss/FBt1/assignment_loss0.03897794087727865 /loss/FRt2/detection_loss2.2605695366104945 1loss/SRqqt2/detection_loss1.0626593781917384 /loss/FBt2/detection_loss0.7588886364453262 % _timestamp1.7858024258528917e+09k' 2loss/SRqqt1/assignment_loss0.8011794430237975 2loss/SRqqt2/assignment_loss0.8403404908551072 3loss/FBt2/assignment_loss1.922730201576703e-08 /loss/FRt1/detection_loss2.3394451899373014 /loss/FRt1/assignment_loss2.667590119268583 %loss/total_loss14.93563771550685 trainer/global_step16049 1loss/SRqqt1/detection_loss0.8755761191440636 /loss/FBt1/detection_loss1.3043279108177457 epoch0 1loss/FBt1/assignment_loss0.03897794087727865 /loss/FRt2/detection_loss2.2605695366104945 1loss/SRqqt2/detection_loss1.0626593781917384 /loss/FBt2/detection_loss0.7588886364453262 % _timestamp1.7858024258528917e+09 _runtime7138.675780272 _step1283 0loss/FRt2/assignment_loss1.9860829311081125 b'(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp41.0 gpu.0.powerWatts117.675' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent39.225 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0Қf'(:  V disk.nvme0n1.in 4.64453125 disk.drbd1673.in0 disk.drbd1931.out0! proc.memory.rssMB 11260.40625 disk.drbd1552.out0 disk.drbd2118.in0 disk.drbd1226.in0 disk.rbd0.in0 disk.drbd1226.out0 disk.drbd1817.out0 disk.rbd1.out 28.7138671875 disk.drbd2041.in0 disk.drbd1552.in0 disk.drbd1828.out0 disk.drbd1938.out0 disk.drbd2078.in0 disk.drbd1584.out0 disk.drbd1965.in0$ memory_percent41.26179814338684 disk.drbd1285.in0 disk.drbd1285.out0 disk.drbd1356.out0 disk.sda.out 193.5234375 disk.drbd1817.in0 disk.sda2.out 193.5234375 disk.drbd2111.out0 disk.drbd2041.out0 disk.drbd2111.in0$ disk./.usageGB578.3530311584473 disk.drbd1938.in0 disk.drbd1510.out0 disk.drbd1584.in0 disk.drbd1289.out0 disk.sda.in1.59375 disk.drbd2078.out0 disk.drbd2118.out0* proc.memory.percent17.182016372680664 disk.drbd1289.in0$ disk.nvme0n1.out297.27978515625 disk.drbd1931.in0 disk.drbd1510.in0 disk.drbd1356.in0* proc.memory.availableMB38494.66796875 cpu13.72959390416352 disk.drbd1828.in0! disk.dm-0.out297.27978515625 disk.rbd1.in0 disk.drbd1965.out0 disk.drbd1673.out0) disk./.usagePercent32.34452915246235 disk.rbd0.out0 disk.sda2.in1.59375 network.recv806667 disk.sda1.out0 disk.dm-0.in 4.64453125 proc.cpu.threads29 network.sent 1.4811404e+07 disk.drbd1550.in0 disk.drbd1550.out0 disk.sda1.in0'8'(: U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts130.374' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent43.458 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0pi'(:  U* proc.memory.availableMB38495.08203125$ disk./.usageGB578.3529472351074 disk.rbd1.out 28.7763671875 disk.drbd1965.out0 disk.drbd1673.in0$ memory_percent41.26116633415222 network.recv808090 disk.drbd1510.out0! disk.dm-0.out297.47900390625 disk.drbd2111.out0 disk.drbd1965.in0 disk.drbd1673.out0 disk.drbd1938.out0 disk.drbd1226.out0 disk.drbd2111.in0 disk.drbd1817.out0* proc.memory.percent17.182016372680664 cpu13.729239556240195 disk.drbd1289.in0 disk.drbd1584.in0 disk.drbd2118.out0 disk.drbd1817.in0 disk.drbd1828.in0 disk.drbd2078.in0 disk.drbd1584.out0 disk.rbd1.in0 disk.drbd1552.in0 disk.drbd2078.out0 disk.nvme0n1.in 4.64453125$ disk.nvme0n1.out297.47900390625 disk.drbd1828.out0 disk.rbd0.out0 disk.sda1.in0 disk.sda1.out0 disk.drbd1226.in0 disk.sda.in1.59375 disk.drbd1356.out0 disk.drbd1931.in0! proc.memory.rssMB 11260.40625 disk.drbd1289.out0 disk.sda2.in1.59375 disk.drbd2041.out0 disk.drbd1285.in0) disk./.usagePercent32.34452445902985 disk.drbd1550.in0 disk.sda.out 194.08984375 disk.drbd1938.in0 disk.drbd1931.out0 proc.cpu.threads29 disk.drbd2041.in0 disk.rbd0.in0 disk.drbd1510.in0 disk.dm-0.in 4.64453125 disk.sda2.out 194.08984375 network.sent 1.4856435e+07 disk.drbd1552.out0 disk.drbd1285.out0 disk.drbd1356.in0 disk.drbd1550.out0 disk.drbd2118.in0 `X2'- _wandbruntime7160  _runtime7160h' _runtime7160.264711671 _step1284 ' lr-AdamW/pg10.00041490643022694287 ' lr-AdamW/pg20.00041490643022694287 trainer/global_step16099 % _timestamp1.7858024474417267e+09GQ' ' lr-AdamW/pg20.00041490643022694287 trainer/global_step16099 % _timestamp1.7858024474417267e+09 _runtime7160.264711671 _step1284 ' lr-AdamW/pg10.00041490643022694287 OMT2'- _wandbruntime7160  _runtime7160\_6''O A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated291590128155648 trainer/global_step16099 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 276979828 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed283653001155468 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 170022949 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current247 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated290555866497024 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 109536619 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed276389410069504 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 106956877 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated290555866497024 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed290555762028032 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated276389427108864 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed291589970480640 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 106956877 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed276389410069504 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed290555762028032 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 _runtime7160.269694516 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14131682125996 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 170013293 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed46148524 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 170013293 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 106956879 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14166351958528 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 170022949 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current249 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 106956879 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 276970170 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14131767929120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed269521319029472 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed63388095 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14549565665792 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 276970170 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14131767929120 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed269521319029472 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed63388095 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14549565665792 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated290555866497024 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed290555762028032 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated276389427108864 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed291589970480640 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 106956877 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed276389410069504 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed290555762028032 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 _runtime7160.269694516 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 7'N2'- _wandbruntime7160  _runtime7160z'M SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 170033486 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed276406576832512 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated290573912886784 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated290573912886784 VCDeviceStatsMonitor.on_train_batch_end/active_byt&~S"es.small_pool.freed14167231578112 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated276406593871872 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 106963520 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 106963522 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 106963520 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 109543411 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current255 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14550466141184 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed269538059201792 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14132645402454 _runtime7160.653510829 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 170033486 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated14167319014912 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current257 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 _step1286 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14167231578112 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated269538076241152 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 276997008 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated291608239934976 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 170023816 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed290573808410624 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 276987336 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 106963522 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated63392034 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed291608078072832 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14132559599218 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 % _timestamp1.7858024478301408e+09 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 170023816 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 30219960329b&'M H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 276997008 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed276406576832512 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 109543668 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated277057636835328 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed283670618801010 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed290573808410624 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 276987336 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 106963522 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated63392034 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed291608078072832 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14132559599218 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 % _timestamp1.7858024478301408e+09 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 170023816 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 170033486 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed276406576832512 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated290573912886784 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated290573912886784 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed14167231578112 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated276406593871872 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 106963520 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 106963522 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 106963520 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 109543411 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current255 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14550466141184 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed269538059201792 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14132645402454 _runtime7160.653510829 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 170033486 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated14167319014912 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current257 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 _step1286 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14167231578112 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated269538076241152 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 276997008 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated291608239934976 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 170023816 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 FDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 trainer/global_step16149 % _timestamp1.7858024692734442e+09 _step1289 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14175656392628 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 170541295 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated277247765259264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14210435523584 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14210348093952 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed63584945 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated291458200782848 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed291458096313856 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated270358344684832 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed277900760654848 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed284533898234976 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14210435523584 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed270358327645472 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 277839980 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated291458200782848 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14594781912576 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current264 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 107289029 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated46292258 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed14210348093952 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 107289029 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 107289027 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current262 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 170541295 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed46291996 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated277900785558528 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed291458096313856 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0܆U''O R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed291458096313856 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated277247765259264 _runtime7182.096730906 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed277247748219904 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 170550951 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 109876941 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated14594920975360 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated284534001077460 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 163966464 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed292495542567424 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14175570589504 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 107289027 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 trainer/global_step16149 % _timestamp1.7858024692734442e+09 _step1289 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14175656392628 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 170541295 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated277247765259264 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14210435523584 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14210348093952 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed63584945 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated291458200782848 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed291458096313856 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated270358344684832 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed277900760654848 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed284533898234976 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14210435523584 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed270358327645472 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 277839980 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated291458200782848 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14594781912576 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current264 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 139062784 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 107289029 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated46292258 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed14210348093952 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 107289029 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 107289027 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current262 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 170541295 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed46291996 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated277900785558528 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14176448062726 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated284551618723114 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated63588884 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed292513655773184 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 _step1290 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 107295672 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14176533865962 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 107295670 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed284551515880518 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated270375084857152 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 % _timestamp1.7858024696508873e+09 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 109884300 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated291476247172608 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated291476247172608 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated46295416 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated292513813441024 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated277917992675328 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 109884046 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed63588882 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 277857160 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 170551818 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14595688001536 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current254 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated14211315150336 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak862781448 &'M ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 107295670 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed284551515880518 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated270375084857152 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 % _timestamp1.7858024696508873e+09 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 109884300 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated291476247172608 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated291476247172608 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated46295416 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated292513813441024 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated277917992675328 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 109884046 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed63588882 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 277857160 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 170551818 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14595688001536 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current254 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated14211315150336 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 277847488 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 170551818 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 170561488 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14176448062726 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated284551618723114 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated63588884 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed292513655773184 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 _step1290 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 107295672 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14176533865962 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 #2'- _wandbruntime7182  _runtime7182cdO' % _timestamp1.7858024696528215e+09 _step1291 /loss/FRt1/detection_loss2.5147512898633355 /loss/FBt1/detection_loss1.2868537219322247 2loss/SRqqt1/assignment_loss0.6633146623782239 .loss/FRt2/detection_loss2.362172676856259 /loss/FBt2/detection_loss0.6244539671192447 _runtime7182.475656671 0loss/FRt2/assignment_loss1.8903439547642167 1loss/SRqqt2/assignment_loss0.704428784885736 3loss/FBt1/assignment_loss5.183012280232778e-09 loss/FBt2/assignment_loss0 &loss/total_loss14.551005788500982 trainer/global_step16149 /loss/FRt1/assignment_loss2.366916817112973 1loss/SRqqt1/detection_loss1.0112358762326275 1loss/SRqqt2/detection_loss1.1265340321731285 epoch0I8' /loss/FBt2/detection_loss0.6244539671192447 _runtime7182.475656671 0loss/FRt2/assignment_loss1.8903439547642167 1loss/SRqqt2/assignment_loss0.704428784885736 3loss/FBt1/assignment_loss5.183012280232778e-09 loss/FBt2/assignment_loss0 &loss/total_loss14.551005788500982 trainer/global_step16149 /loss/FRt1/assignment_loss2.366916817112973 1loss/SRqqt1/detection_loss1.0112358762326275 1loss/SRqqt2/detection_loss1.1265340321731285 epoch0 % _timestamp1.7858024696528215e+09 _step1291 /loss/FRt1/detection_loss2.5147512898633355 /loss/FBt1/detection_loss1.2868537219322247 2loss/SRqqt1/assignment_loss0.6633146623782239 .loss/FRt2/detection_loss2.362172676856259 \'(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts129.717' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent43.239000000000004 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0kvR'(:  W disk.drbd1817.out0 disk.drbd1673.in0 disk.-&xsda.in1.59375 disk.drbd2078.out0# disk.nvme0n1.out298.2568359375 disk.drbd1552.in0 disk.sda.out 194.64453125 disk.drbd1510.out0 disk.drbd1226.out0 disk.drbd1938.in0* proc.memory.availableMB38495.18359375 disk.drbd1931.in0 disk.drbd1938.out0 disk.sda1.in0$ disk./.usageGB578.3530883789062 disk.nvme0n1.in 4.64453125 disk.drbd1356.out0 disk.drbd2111.out0 disk.drbd2118.in0 disk.drbd1828.out0 disk.drbd1289.in0 disk.dm-0.out298.2568359375 disk.drbd1584.out0 disk.sda2.in1.59375 disk.drbd1552.out0 disk.drbd2078.in0 disk.rbd1.out 28.80859375 network.recv810628 disk.drbd1285.in0! proc.memory.rssMB 11260.40625* proc.memory.percent17.182016372680664 disk.drbd1356.in0% memory_percent41.261011362075806 disk.drbd2111.in0 disk.drbd1673.out0 disk.rbd0.in0) disk./.usagePercent32.34453235252996 disk.dm-0.in 4.64453125 disk.drbd2118.out0 disk.rbd1.in0 disk.drbd2041.out0 disk.drbd1289.out0 disk.rbd0.out0 network.sent 1.4904492e+07 disk.drbd1828.in0 disk.drbd1550.in0 disk.drbd1226.in0 disk.drbd1584.in0 disk.drbd1817.in0 disk.drbd1965.in0 disk.drbd1931.out0 disk.drbd2041.in0 disk.drbd1550.out0 disk.drbd1510.in0 disk.sda2.out 194.64453125 disk.drbd1965.out0 disk.sda1.out0 disk.drbd1285.out0 cpu13.728698814011613 proc.cpu.threads299'(: տU gpu.0.gpu1.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts123.168' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent41.056000000000004 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0JZe'(:  V disk.drbd2078.out0 disk.sda2.in1.59375 disk.drbd1552.out0 network.recv812144 disk.drbd1285.in0 disk.drbd2111.out0 disk.drbd1965.in0 disk.drbd1931.in0 cpu13.728368863757202 proc.cpu.threads29 disk.sda.out 195.19921875 disk.drbd1938.out0 disk.sda1.out0( disk./.usagePercent32.3445314991786 disk.drbd1356.in0 disk.drbd1817.in0 disk.sda1.in0 disk.drbd1285.out0 disk.drbd1356.out0 disk.rbd1.out 28.90234375* proc.memory.percent17.182016372680664 disk.drbd1510.in0 disk.drbd2078.in0 disk.drbd1289.in0 disk.sda.in1.59375 disk.drbd2118.in0 disk.nvme0n1.in 4.64453125! disk.dm-0.out299.97412109375 disk.sda2.out 195.19921875 disk.drbd1817.out0 disk.drbd1965.out0 disk.drbd2041.out0 disk.drbd1552.in0 disk.drbd1828.out0 disk.drbd1550.in0 disk.rbd0.in0 disk.dm-0.in 4.64453125 disk.drbd1584.out0$ disk.nvme0n1.out299.97412109375 disk.drbd1931.out0 disk.drbd1828.in0 disk.rbd0.out0 disk.drbd1289.out0 disk.drbd2118.out0 disk.drbd2111.in0% memory_percent41.261839866638184 disk.drbd1550.out0 disk.rbd1.in0 disk.drbd1673.in0 disk.drbd1673.out0( proc.memory.availableMB 38494.640625! proc.memory.rssMB 11260.40625 network.sent 1.4949499e+07$ disk./.usageGB578.3530731201172 disk.drbd1938.in0 disk.drbd1584.in0 disk.drbd2041.in0 disk.drbd1510.out0 disk.drbd1226.in0 disk.drbd1226.out02'- _wandbruntime7204  _runtime7204QY' ' lr-AdamW/pg20.00041748364887547344 trainer/global_step16199 % _timestamp1.7858024913527985e+09 _runtime7204.175763708 _step1292 ' lr-AdamW/pg10.00041748364887547344[' % _timestamp1.7858024913527985e+09 _runtime7204.175763708 _step1292 ' lr-AdamW/pg10.00041748364887547344 ' lr-AdamW/pg20.00041748364887547344 trainer/global_step16199  2(- _wandbruntime7204  _runtime7204b>'(O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed293401067911168 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14254431659008 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14219544856136 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 107621179 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 107621177 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current243 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 _runtime7204.177023874 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed46434642 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated293401223489024 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed278761116494848 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated285414898156968 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated63781797 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed278106086370304 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed14254344229376 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed292360430599680 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 171069297 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed271195336261472 _step1293 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 110216682 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14254431659008 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated292360535068672 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14219459053012 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 $ _timestamp1.785802491353749e+09 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 171078953 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 278690474 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 278700132 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed285414795314484 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 107621179 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated46434885 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current245 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated271195353300832 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14639951416320 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated278106103409664 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14254344229376 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 107621177 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed278106086370304 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed2923604305996804j'(O UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 171078953 trainer/global_step16199 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 171069297 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 110216682 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14254431659008 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated292360535068672 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14219459053012 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 $ _timestamp1.785802491353749e+09 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 171078953 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 278690474 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 278700132 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed285414795314484 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 107621179 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated46434885 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current245 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated271195353300832 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14639951416320 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated278106103409664 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14254344229376 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 107621177 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed278106086370304 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed292360430599680 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed293401067911168 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14254431659008 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14219544856136 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 107621179 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 107621177 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current243 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 _runtime7204.177023874 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed46434642 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated293401223489024 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed278761116494848 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated285414898156968 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated63781797 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed278106086370304 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed14254344229376 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed292360430599680 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 171069297 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed271195336261472 _step1293 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 JDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 171089490 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 107627820 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated278778348515328 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 107627820 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed278123253133312 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed285432412960026 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14220336526234 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated271212093473152 $ _timestamp1.785802491715807e+09 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 171089490 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated278123270172672 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 278707640 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated14255311285760 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 trainer/global_step16199 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed293419177223680 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 171079820 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current255 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 110223199 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated63785734 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14220422329470 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14255223848960 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.alHu=Plocated46437720 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 107627822 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 278717312 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed278778323611648 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated285432515802622 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392t&(M O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 110223199 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated63785734 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14220422329470 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14255223848960 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated46437720 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 107627822 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 278717312 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed278778323611648 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated285432515802622 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated292378581458432 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated278123270172672 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 110223454 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 171089490 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 107627820 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated278778348515328 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 107627820 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed278123253133312 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed285432412960026 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14220336526234 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated271212093473152 $ _timestamp1.785802491715807e+09 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 171089490 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated278123270172672 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 278707640 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated14255311285760 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 trainer/global_step16199 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed293419177223680 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 171079820 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current255 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 FDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 171606955 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14298427794432 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated63978647 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current269 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated272032361916832 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 _runtime7225.980119465 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated286295795236476 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 171597299 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated294306745567744 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 $ _timestamp1.785802513156804e+09 _step1297 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 110558840 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed46579924 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed272032344877472 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed278964424520704 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current271 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14263433319644 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14298427794432 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed293262764885504 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed293262764885504 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 107953329 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14298340364800 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated278964441560064 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed14298340364800 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed294306592087040 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated279621497238528 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 171606955 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14298427794432 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated63978647 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current269 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated272032361916832 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 _runtime7225.980119465 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated286295795236476 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 171597299 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated294306745567744 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 UGD _eviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 $ _timestamp1.785802513156804e+09 _step1297 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 110558840 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed46579924 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed272032344877472 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed278964424520704 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current271 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14263433319644 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14298427794432 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed293262764885504 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed293262764885504 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 107953329 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14298340364800 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated278964441560064 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed14298340364800 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed294306592087040 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated279621497238528 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated293280915744256 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated278981608323072 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 171617492 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14264224989742 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14686019179008 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 171607822 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14264310792978 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed278981591283712 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 279567792 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 171617492 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current280 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated294324856298496 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 107959970 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed63982582 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current278 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 $ _timestamp1.785802513539137e+09 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 279567792 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed294324698630656 _runtime7226.362507366 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 279577464 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 279577464 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed286313310039534 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated286313412882130 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 110565422 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated14686151943168 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 107959970 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed278981591283712 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed279638679451648 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14299219984384 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed293280811268096 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated14299307421184 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 107959972 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0&(M ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14686019179008 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 171607822 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14264310792978 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed278981591283712 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 279567792 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 171617492 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current280 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated294324856298496 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 107959970 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed63982582 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current278 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 $ _timestamp1.785802513539137e+09 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 279567792 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed294324698630656 _runtime7226.362507366 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 279577464 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 279577464 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed286313310039534 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated286313412882130 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 110565422 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated14686151943168 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 107959970 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed278981591283712 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed279638679451648 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14299219984384 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed293280811268096 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated14299307421184 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 107959972 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated14299307421184 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated46583118 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 107959972 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated293280915744256 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated278981608323072 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 171617492 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14264224989742 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 2(- _wandbruntime7226  _runtime7226 ? ( 2loss/SRqqt1/assignment_loss0.6601555095553132 4loss/FBt2/assignment_loss1.4798375747792807e-07 0loss/SRqqt2/detection_loss1.053017854386525 epoch0 0loss/FRt1/assignment_loss2.3501653937069147 .loss/FRt2/detection_loss2.411865814961493 trainer/global_step16249 _step1299 3loss/FBt1/assignment_loss9.169944803488761e-09 1loss/SRqqt1/detection_loss1.0855901756057782 /loss/FBt1/detection_loss0.8684229797755296 /loss/FBt2/detection_loss0.8964569191207531 % _timestamp1.7858025135406196e+09 _runtime7226.363598191 /loss/FRt2/assignment_loss1.888734509795904 2loss/SRqqt2/assignment_loss0.7475578047783491 /loss/FRt1/detection_loss2.6494402851204613 &loss/total_loss14.611407403960726yy( 1loss/SRqqt1/detection_loss1.0855901756057782 /loss/FBt1/detection_loss0.8684229797755296 /loss/FBt2/detection_loss0.8964569191207531 % _timestamp1.7858025135406196e+09 _runtime7226.363598191 /loss/FRt2/assignment_loss1.888734509795904 2loss/SRqqt2/assignment_loss0.7475578047783491 /loss/FRt1/detection_loss2.6494402851204613 &loss/total_loss14.611407403960726 2loss/SRqqt1/assignment_loss0.6601555095553132 4loss/FBt2/assignment_loss1.4798375747792807e-07 0loss/SRqqt2/detection_loss1.053017854386525 epoch0 0loss/FRt1/assignment_loss2.3501653937069147 .loss/FRt2/detection_loss2.411865814961493 trainer/global_step16249 _step1299 3loss/FBt1/assignment_loss9.169944803488761e-09 Ǜ((: U gpu.0.gpu32.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts123.055' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent41.01833333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0>c((:  V disk.drbd1938.out0 disk.drbd1817.in0 disk.sda.in1.59375 disk.drbd1828.in0 disk.drbd1817.out0 disk.drbd2041.out0 disk.sda.out 196.03515625! proc.memory.rssMB 11260.40625 disk.drbd2118.in0( proc.memory.availableMB 38495.328125 disk.drbd1931.out0) disk./.usagePercent32.34452723242178 disk.drbd1584.out0 disk.drbd1289.in0 disk.drbd1285.out0 disk.rbd1.in0$ disk./.usageGB578.3529968261719 disk.rbd1.out 29.05078125 disk.drbd1673.out0 disk.drbd1550.in0 disk.drbd1938.in0 disk.drbd1510.in0* proc.memory.percent17.182016372680664 disk.drbd1931.in0 disk.drbd1552.in0 disk.drbd1550.out0 disk.sda1.in0$ memory_percent41.26079082489014 network.sent 1.4997596e+07 disk.drbd2078.in0 disk.drbd1552.out0 cpu13.727718924637156 network.recv814807 disk.drbd2111.out0 disk.drbd2041.in0 disk.sda1.out0 disk.drbd1285.in0 proc.cpu.threads29 disk.drbd2078.out0 disk.drbd1673.in0 disk.rbd0.in0 disk.rbd0.out0 disk.drbd1226.in0 disk.drbd1226.out0 disk.dm-0.in 4.64453125 disk.drbd2118.out0 disk.drbd1356.out0 disk.sda2.in1.59375 disk.sda2.out 196.03515625 disk.drbd2111.in0 disk.drbd1584.in0 disk.drbd1356.in0 disk.drbd1965.in0 disk.drbd1965.out0 disk.drbd1510.out0# disk.nvme0n1.out301.1162109375 disk.drbd1828.out0 disk.drbd1289.out0 disk.dm-0.out301.1162109375 disk.nvme0n1.in 4.644531253]!Ed((:  U) disk./.usagePercent32.34452829911098 disk.sda1.in0 disk.drbd1226.out0 disk.drbd2078.out0 disk.drbd1356.out0 disk.drbd2111.in0 disk.rbd1.out 29.0830078125 disk.drbd1673.in0 disk.drbd1285.in0 disk.rbd0.out0 disk.drbd1965.in0 disk.drbd1584.out0 disk.drbd2118.in0 disk.rbd1.in0 disk.drbd1289.in0 disk.sda.out 196.38671875 disk.nvme0n1.in 4.64453125 disk.drbd1356.in0 disk.drbd1817.in0 disk.drbd1828.in0 disk.drbd2041.in0 disk.drbd1550.out0 disk.drbd1938.out0 disk.drbd1510.out0 disk.drbd1510.in0 disk.drbd1965.out0 proc.cpu.threads29 disk.drbd2041.out0 cpu13.72749398207515$ disk./.usageGB578.3530158996582# disk.nvme0n1.out301.2724609375 disk.drbd2111.out0 disk.sda1.out0 disk.drbd2118.out0 disk.dm-0.in 4.64453125 disk.sda2.in1.59375 disk.drbd1817.out0$ memory_percent41.26002788543701 disk.drbd1828.out0 network.recv816422( proc.memory.availableMB 38495.828125 disk.drbd1289.out0 disk.drbd1584.in0 disk.drbd1552.out0 network.sent 1.5042707e+07 disk.drbd1931.in0 disk.drbd1938.in0! proc.memory.rssMB 11260.40625 disk.drbd1673.out0 disk.drbd1550.in0 disk.sda.in1.59375 disk.drbd1285.out0 disk.dm-0.out301.2724609375 disk.drbd1226.in0 disk.drbd2078.in0 disk.sda2.out 196.38671875 disk.drbd1931.out0 disk.drbd1552.in0 disk.rbd0.in0* proc.memory.percent17.182016372680664_ ˛((: V gpu.0.gpu32.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts97.908' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent32.635999999999996 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0`e2(- _wandbruntime7248  _runtime7248_R( ' lr-AdamW/pg10.00042006086752400406 ' lr-AdamW/pg20.00042006086752400406 trainer/global_step16299 $ _timestamp1.785802535224062e+09 _runtime7248.047036887 _step1300+G( ' lr-AdamW/pg10.00042006086752400406 ' lr-AdamW/pg20.00042006086752400406 trainer/global_step16299 $ _timestamp1.785802535224062e+09 _runtime7248.047036887 _step1300 2(- _wandbruntime7248  _runtime7248(~ (O JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated295212269944320 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated279822779710464 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 280420436 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed287176589473500 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14342336500224 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed14342336500224 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed295212122755072 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 _step1301 Q:6dCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 108285479 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 110899320 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 _runtime 7248.04832189 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 280410778 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed279822762671104 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14307321783152 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated294165203640320 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated64175497 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated46723823 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed272869353493472 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 172125301 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed64175495 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14307235980028 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 trainer/global_step16299 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed294165099171328 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14342423929856 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 110899066 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current252 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 122285568 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated272869370532832 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 108285479 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated294165203640320 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed294165099171328 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14342423929856 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14730294580224 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 108285477 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 147189248 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 '(O OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14342336500224 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed14342336500224 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed295212122755072 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 _step1301 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 108285479 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 110899320 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 _runtime 7248.04832189 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 280410778 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed279822762671104 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14307321783152 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated294165203640320 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated64175497 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated46723823 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed272869353493472 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 172125301 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed64175495 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14307235980028 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 trainer/global_step16299 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed294165099171328 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14342423929856 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 110899066 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current252 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 122285568 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated272869370532832 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 108285479 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated294165203640320 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed294165099171328 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14342423929856 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14730294580224 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 108285477 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 147189248 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated295212269944320 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated279822779710464 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 280420436 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed287176589473500 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 22(- _wandbruntime7248  _runtime7248&(M KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated14731330260992 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 % _timestamp1.7858025355901778e+09 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 108292122 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated294183250030080 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated294183250030080 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed46726379 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 110905811 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14731193302528 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 108292120 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14343216119808 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 108292122 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 _runtime7248.413514271 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed279839929434112 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 280437616 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 172135824 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated279839946473472 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed14343216119808 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 280437616 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 172135824 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated64179434 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed64179432 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated287194309961638 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current255 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated279839946473472 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 280427944 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated295230390456320 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed295230228594176 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated46726634 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed280499035291648 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14308113453250 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed272886093665792 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 172145494 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 280427944 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 108292120 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated287194309961638 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current255 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated279839946473472 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 280427944 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated295230390456320 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed295230228594176 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated46726634 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed280499035291648 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14308113453250 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed272886093665792 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 172145494 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 280427944 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 108292120 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 % _timestamp1.7858025355901778e+09 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 108292122 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated294183250030080 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated294183250030080 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed46726379 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 110905811 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14731193302528 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 108292120 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14343216119808 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 108292122 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 _runtime7248.413514271 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed279839929434112 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 280437616 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 172135824 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated279839946473472 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed14343216119808 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 280437616 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 172135824 C462(- _wandbruntime7248  _runtime7248( 2loss/SRqqt2/assignment_loss0.7041906203974226 1loss/SRqqt1/detection_loss1.0925917336911273 trainer/global_step16299 /loss/FRt2/assignment_loss2.715641542326046 2loss/SRqqt1/assignment_loss0.7105709149052726 3loss/FBt2/assignment_loss4.860040252634252e-07 -loss/FRt1/detection_loss2.33631526584688 /loss/FRt2/detection_loss2.4533889648280565 .loss/FBt1/detection_loss0.633119010925293 /loss/FBt2/detection_loss0.8086510160311053 &loss/total_loss14.768392196893894 /loss/FRt1/assignment_loss2.103482784095563 loss/FBt1/assignment_loss0 1loss/SRqqt2/detection_loss1.2104398578431013 % _timestamp1.7858025355919285e+09 _runtime7248.414749348 _step1303 epoch0( 1loss/SRqqt1/detection_loss1.0925917336911273 trainer/global_step16299 /loss/FRt2/assignment_loss2.715641542326046 2loss/SRqqt1/assignment_loss0.7105709149052726 3loss/FBt2/assignment_loss4.860040252634252e-07 -loss/FRt1/detection_loss2.33631526584688 /loss/FRt2/detection_loss2.4533889648280565 .loss/FBt1/detection_loss0.633119010925293 /loss/FBt2/detection_loss0.8086510160311053 &loss/total_loss14.768392196893894 /loss/FRt1/assignment_loss2.103482784095563 loss/FBt1/assignment_loss0 1loss/SRqqt2/detection_loss1.2104398578431013 % _timestamp1.7858025355919285e+09 _runtime7248.414749348 _step1303 epoch0 2loss/SRqqt2/assignment_loss0.7041906203974226 |((: U gpu.0.gpu32.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts128.754' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.91799999999999 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0b+a((:  V* disk./.usagePercent32.344523179002806 disk.drbd1550.out0 disk.drbd1510.in0 disk.dm-0.out301.8037109375 disk.drbd2111.in0 disk.drbd1817.out0 disk.drbd1673.out0 disk.drbd2118.in0 disk.drbd2118.out0 disk.rbd1.in0# memory_percent41.2616491317749 disk.rbd0.in0 disk.sda1.out0 disk.drbd1965.out0 disk.drbd2041.in0 disk.drbd1817.in0 network.sent 1.5045837e+07 disk.drbd1552.in0 disk.drbd1552.out0 disk.drbd2078.in0 disk.sda1.in0 disk.drbd1226.out0 disk.drbd1584.in0 disk.sda2.in1.59375 disk.sda2.out 196.765625 disk.drbd1828.in0 disk.drbd1938.in0 disk.rbd1.out 29.1767578125 disk.drbd1965.in0 disk.drbd1931.in0 disk.drbd1289.out0 disk.sda.out 196.765625 disk.nvme0n1.in 4.64453125 disk.drbd1931.out0 disk.drbd2041.out0 disk.rbd0.out0! proc.memory.rssMB 11260.40625 disk.drbd1289.in0 disk.drbd1938.out0 disk.drbd1356.in0 proc.cpu.threads29 disk.drbd1550.in0 disk.drbd1285.in0 cpu13.727147204019756 disk.drbd1673.in0 disk.drbd1828.out0 disk.drbd2078.out0 disk.drbd1584.out0$ disk./.usageGB578.3529243469238 disk.drbd1285.out0 disk.drbd1226.in0* proc.memory.percent17.182016372680664 network.recv817337 disk.sda.in1.59375 disk.dm-0.in 4.64453125 disk.drbd1356.out0( proc.memory.availableMB 38494.765625 disk.drbd1510.out0# disk.nvme0n1.out301.8037109375 disk.drbd2111.out072(- _wandbruntime7269  _runtime7269y( % _timestamp1.7858025571705089e+09 _runtime7269.993473641 _step1304 & lr-AdamW/pg10.0004213494768482694 & lr-AdamW/pg20.0004213494768482694 trainer/global_step163494( & lr-AdamW/pg20.0004213494768482694 trainer/global_step16349 % _timestamp1.7858025571705089e+09 _runtime7269.993473641 _step1304 & lr-AdamW/pg10.0004213494768482694 rAS2(- _wandbruntime7269  _runtime7269&[`'(O GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed14386332635648 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14386332635648 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed288057486553008 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated273706379148832 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14351124443536 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14351210246660 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 108617627 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current261 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed64372345 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 172662959 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 111239783 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed296117665583104 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated281342208918528 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 108617627 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated296117827452416 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated280681117860864 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 281280588 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated64372347 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 281270930 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14386420065280 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14386420065280 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed295067433457152 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 172662959 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 172653303 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 trainer/global_step16349 % _timestamp1.7858025571714807e+09 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed280681100821504 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14775481568256 _step1305 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed295067433457152 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 108617629 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated288057589395492 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current259 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed281342184014848 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 172653303 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed280681100821504 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed273706362109472 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 _runtime7269.994768198 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated295067537926144 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 108617629)y('(O \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed273706362109472 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 _runtime7269.994768198 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated295067537926144 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 108617629 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed14386332635648 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14386332635648 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed288057486553008 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated273706379148832 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14351124443536 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14351210246660 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 108617627 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current261 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed64372345 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 172662959 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 111239783 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed296117665583104 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated281342208918528 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 108617627 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated296117827452416 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated280681117860864 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 281280588 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated64372347 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 281270930 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14386420065280 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14386420065280 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed295067433457152 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 172662959 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 172653303 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 trainer/global_step16349 % _timestamp1.7858025571714807e+09 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed280681100821504 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14775481568256 _step1305 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed295067433457152 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 108617629 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated288057589395492 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current259 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed281342184014848 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 172653303 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed280681100821504 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 >y2(- _wandbruntime7270  _runtime7270< (M P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14352087719994 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 172673496 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated296135933376512 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 281297768 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 281288096 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed46870307 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed281359391131648 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current260 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _]runtime7270.383658285 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed14387212255232 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 149279232 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated288075207041146 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 108624270 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed64376282 % _timestamp1.7858025575602977e+09 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14776392965632 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 172673496 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated14387299692032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 108624272 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 108624272 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed295085479839744 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 108624270 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14352001916758 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated295085584315904 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated46870565 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 124375552 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 281297768 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated64376284 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed273723102281792 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated280698284623872 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed280698267584512 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated281359416035328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 111246589 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated14776517341184 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14387212255232 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 172663826 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 111246589 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated14776517341184 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14387212255232 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 172663826 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14352087719994 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 172673496 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated296135933376512 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 281297768 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 281288096 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed46870307 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed281359391131648 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current260 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _runtime7270.383658285 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed14387212255232 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 149279232 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated288075207041146 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 108624270 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed64376282 % _timestamp1.7858025575602977e+09 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14776392965632 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 172673496 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated14387299692032 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 108624272 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 108624272 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed295085479839744 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 108624270 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14352001916758 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated295085584315904 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated46870565 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 124375552 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 281297768 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated64376284 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed273723102281792 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated280698284623872 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 v2(- _wandbruntime7270  _runtime7270A$( &loss/total_loss14.004258411681533 trainer/global_step16349 _runtime 7270.38479993 _step1307 /loss/FBt2/detection_loss0.9312811959115789 /loss/FRt1/assignment_loss2.062782580011031 0loss/FRt2/assignment_loss1.9638706077086299 2loss/SRqqt1/assignment_loss0.6468703948904748 /loss/FRt2/detection_loss2.4018905580926098 1loss/SRqqt1/detection_loss1.1406272379620992 0loss/FBt1/detection_loss0.47483939734789044 epoch0 .loss/FRt1/detection_loss2.461938230868648 0loss/SRqqt2/detection_loss1.095586655048542 % _timestamp1.7858025575619385e+09 2loss/SRqqt2/assignment_loss0.8142776612180452 loss/FBt1/assignment_loss0 2loss/FBt2/assignment_loss0.010293892621984924a$h( &loss/total_loss14.004258411681533 trainer/global_step16349 _runtime 7270.38479993 _step1307 /loss/FBt2/detection_loss0.9312811959115789 /loss/FRt1/assignment_loss2.062782580011031 0loss/FRt2/assignment_loss1.9638706077086299 2loss/SRqqt1/assignment_loss0.6468703948904748 /loss/FRt2/detection_loss2.4018905580926098 1loss/SRqqt1/detection_loss1.1406272379620992 0loss/FBt1/detection_loss0.47483939734789044 epoch0 .loss/FRt1/detection_loss2.461938230868648 0loss/SRqqt2/detection_loss1.095586655048542 % _timestamp1.7858025575619385e+09 2loss/SRqqt2/assignment_loss0.8142776612180452 loss/FBt1/assignment_loss0 2loss/FBt2/assignment_loss0.010293892621984924 '((: U gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts133.034' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent44.34466666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0q=w_((:  ɪV disk.drbd1931.out0 disk.drbd1289.in0 disk.rbd0.in0 disk.drbd2078.out0 disk.drbd2111.in0! proc.memory.rssMB 11260.40625 disk.sda1.in0 disk.sda1.out0 disk.drbd1285.out0 disk.drbd1226.out0 disk.drbd1673.in0 disk.drbd2041.in0 disk.drbd1289.out0 disk.dm-0.in 4.64453125 disk.dm-0.out301.9169921875 disk.drbd1510.out0 disk.drbd2118.in0 disk.drbd1938.in0 disk.drbd1673.out0 disk.drbd1931.in0 disk.drbd1552.in0$ disk./.usageGB578.3529968261719 disk.drbd1965.out0 disk.rbd0.out0 disk.nvme0n1.in 4.64453125 disk.drbd1828.in0 disk.drbd1285.in0 disk.drbd1584.in0 disk.drbd1817.out0 disk.drbd1552.out0 disk.drbd1550.out0 network.recv819406) disk./.usagePercent32.34452723242178# disk.nvme0n1.out301.9169921875 disk.drbd2078.in0 disk.sda2.out197.1875 disk.drbd2111.out0 disk.rbd1.in0 disk.drbd1828.out0 cpu13.726818255399378 proc.cpu.threads29 network.sent 1.5091281e+07 disk.drbd1226.in0* proc.memory.percent17.182016372680664 disk.sda.in1.59375 disk.drbd1965.in0 disk.drbd2118.out0 disk.sda2.in1.59375 disk.drbd1510.in0 disk.drbd1356.out0 disk.drbd1817.in0 disk.rbd1.out 29.2392578125$ memory_percent41.26130938529968* proc.memory.availableMB38494.98828125 disk.sda.out197.1875 disk.drbd1584.out0 disk.drbd1356.in0 disk.drbd2041.out0 disk.drbd1550.in0 disk.drbd1938.out0]((: 畕U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts131.597' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent43.86566666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0 b((:  U disk.sda1.in0 disk.drbd1673.out0 disk.drbd1931.in0 disk.drbd1550.in0 disk.sda2.in1.59375 disk.drbd1510.out0 disk.drbd1828.in0 disk.drbd1550.out0 disk.drbd1285.in0 disk.drbd1584.out0( proc.memory.availableMB 38495.234375* proc.memory.percent17.182016372680664 disk.sda.out 197.65234375 disk.drbd2111.in0 disk.drbd1931.out0$ disk./.usageGB578.3529968261719 disk.drbd1817.out0 disk.drbd2118.in0 disk.sda.in1.59375 disk.drbd2078.in0 disk.rbd1.in0 disk.rbd1.out 29.2392578125 network.sent 1.5139568e+07 disk.drbd1285.out0 disk.drbd2118.out0 disk.drbd1965.in0 disk.dm-0.out 302.283203125 disk.drbd1356.in0 disk.drbd1552.out0 disk.drbd1938.in0 disk.rbd0.in0 disk.drbd2111.out0 disk.rbd0.out0 cpu13.726442854687194 proc.cpu.threads29 disk.drbd1356.out0 disk.drbd1510.in0 disk.dm-0.in 4.64453125 disk.drbd1226.in0 disk.drbd1226.out0 disk.drbd1673.in0 disk.drbd2041.in0 disk.drbd1552.in0 disk.drbd1584.in0 disk.sda2.out 197.65234375 disk.drbd1965.out0 disk.drbd1289.in0 disk.drbd1289.out0 network.recv827032 disk.drbd1828.out0 disk.drbd1938.out0 disk.sda1.out0 disk.nvme0n1.in 4.64453125" disk.nvme0n1.out 302.283203125 disk.drbd1817.in0) disk./.usagePercent32.34452723242178 disk.drbd2078.out0 disk.drbd2041.out0! proc.memory.rssMB 11260.40625# memory_percent41.2609338760376:xz2(- _wandbruntime7292  _runtime7292( % _timestamp1.7858025794127598e+09 _runtime7292.235728654 _step1308 & lr-AdamW/pg10.0004226380861725347 & lr-AdamW/pg20.0004226380861725347 trainer/global_step1639908( % _timestamp1.7858025794127598e+09 _runtime7292.235728654 _step1308 & lr-AdamW/pg10.0004226380861725347 & lr-AdamW/pg20.0004226380861725347 trainer/global_step16399 ӥ2(- _wandbruntime7292  _runtime7292=f%(O GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 108949777 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated295969872211968 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated281539456011264 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14430416200704 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current252 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated295969872211968 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated281539456011264 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14820683832320 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed288938383632516 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 108949777 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 111580661 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 145092096 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed281539438971904 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed64569195 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 173190961 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed281539438971904 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14430416200704 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed282202539854848 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14430328771072 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 173181305 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated47011716 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed274543370725472 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated14820804020736 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed295969767742976 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed14430328771072 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14395098710168 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14395012907044 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.HF.allocated 111580913 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current250 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0Lp'(O =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 _step1309 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed295969767742976 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 120188416 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 $ _timestamp1.785802579413716e+09 _runtime7292.236914131 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated295969872211968 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated281539456011264 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14820683832320 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed288938383632516 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 108949777 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 111580661 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 145092096 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed281539438971904 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed64569195 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 173190961 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed281539438971904 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14430416200704 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed282202539854848 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14430328771072 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 173181305 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated47011716 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed274543370725472 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated14820804020736 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed295969767742976 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed14430328771072 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14395098710168 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14395012907044 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 111580913 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current250 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 108949777 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated295969872211968 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated281539456011264 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14430416200704 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current252 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 % _timestamp1.7858025797817123e+09 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14431208390656 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14395890380266 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 173191828 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14821582184960 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed282219746971648 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated64573134 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated295987918601728 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 282157920 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated14821717046272 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 108956420 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 _step1310 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 173201498 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 173191828 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated295987918601728 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 trainer/global_step16399 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated282219771875328 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current261 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed295987814125568 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed274560110897792 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _runtime7292.605057729 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 111587468 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 111587731 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated47014597 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed288956001278058 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 282148248 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 282157920 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 282148248 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 173201498 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated274560127937152 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed281556605734912 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed281556605734912 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032,&(M OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 108956422 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 % _timestamp1.7858025797817123e+09 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14431208390656 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14395890380266 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 173191828 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14821582184960 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed282219746971648 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated64573134 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated295987918601728 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 282157920 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated14821717046272 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 108956420 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 _step1310 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 173201498 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 173191828 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated295987918601728 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 trainer/global_step16399 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated282219771875328 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current261 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed295987814125568 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed274560110897792 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _runtime7292.605057729 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 111587468 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 111587731 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated47014597 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed288956001278058 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 282148248 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 282157920 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 282148248 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 173201498 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated274560127937152 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed281556605734912 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed281556605734912 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14395976183502 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 92(- _wandbruntime7292  _runtime72922Q ( .loss/FRt2/detection_loss2.267955103090831 epoch0 2loss/SRqqt2/assignment_loss0.8226023760114057 -loss/FBt1/detection_loss1.74321123967982 trainer/global_step16399 % _timestamp1.7858025797833354e+09 _runtime7292.606160664 /loss/FRt1/assignment_loss2.305919112364451 4loss/FBt1/assignment_loss1.6556822994099699e-07 _step1311 2loss/SRqqt1/assignment_loss0.7373166649614449 .loss/FRt1/detection_loss2.634396819571654 0loss/SRqqt1/detection_loss1.151114005940156 1loss/SRqqt2/detection_loss1.0462652978021652 /loss/FBt2/detection_loss1.0185694650367454 &loss/total_loss15.741000964589585 0loss/FRt2/assignment_loss2.0136506969020482 4loss/FBt2/assignment_loss1.7660633120528845e-08^( trainer/global_step16399 % _timestamp1.7858025797833354e+09 _runtime7292.606160664 /loss/FRt1/assignment_loss2.305919112364451 4loss/FBt1/assignment_loss1.6556822994099699e-07 _step1311 2loss/SRqqt1/assignment_loss0.7373166649614449 .loss/FRt1/detection_loss2.634396819571654 0loss/SRqqt1/detection_loss1.151114005940156 1loss/SRqqt2/detection_loss1.0462652978021652 /loss/FBt2/detection_loss1.0185694650367454 &loss/total_loss15.741000964589585 0loss/FRt2/assignment_loss2.0136506969020482 4loss/FBt2/assignment_loss1.7660633120528845e-08 .loss/FRt2/detection_loss2.267955103090831 epoch0 2loss/SRqqt2/assignment_loss0.8226023760114057 -loss/FBt1/detection_loss1.74321123967982 G/((:  U disk.rbd1.in0 disk.drbd1931.in0 disk.drbd1552.in0# disk.nvme0n1.out302.5751953125* proc.memory.percent17.182016372680664) disk./.usagePercent32.34452381901633 disk.drbd1510.in0 disk.drbd2078.in0 disk.drbd2111.out0 disk.sda2.Su.out 198.84375( proc.memory.availableMB 38495.703125 disk.drbd1938.in0 cpu13.72608385212529 disk.drbd1552.out0! proc.memory.rssMB 11260.40625 disk.sda.out 198.84375 disk.drbd1356.in0 disk.drbd1356.out0 disk.drbd1965.out0 disk.drbd2041.out0 disk.drbd1828.out0 disk.drbd1285.in0 disk.drbd1673.in0 disk.drbd1550.out0 disk.rbd0.out0 disk.drbd1226.in0 disk.sda2.in1.59375 disk.drbd2111.in0 disk.rbd1.out 29.271484375 disk.drbd1828.in0 disk.drbd1938.out0 disk.drbd2041.in0 disk.drbd1289.out0 disk.rbd0.in0 disk.sda1.in0 disk.drbd1510.out0 disk.dm-0.out302.5751953125 disk.drbd1584.in0 disk.drbd1584.out0 network.sent 1.5184699e+07 disk.drbd1550.in0 disk.nvme0n1.in 4.64453125$ memory_percent41.26021862030029 proc.cpu.threads29 disk.drbd1817.in0 disk.drbd1817.out0 disk.sda1.out0 disk.drbd1226.out0 network.recv828615 disk.drbd1965.in0 disk.dm-0.in 4.64453125$ disk./.usageGB578.3529357910156 disk.drbd1673.out0 disk.drbd2118.out0 disk.drbd1289.in0 disk.drbd1285.out0 disk.drbd2078.out0 disk.drbd2118.in0 disk.drbd1931.out0 disk.sda.in1.59375:e<((: ƟV gpu.0.gpu34.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts132.055' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent44.01833333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors02(- _wandbruntime7314  _runtime7314T -( % _timestamp1.7858026012073421e+09 _runtime7314.031657023 _step1312 ' lr-AdamW/pg10.00042392669549680005 ' lr-AdamW/pg20.00042392669549680005 trainer/global_step16449MQ@( ' lr-AdamW/pg20.00042392669549680005 trainer/global_step16449 % _timestamp1.7858026012073421e+09 _runtime7314.031657023 _step1312 ' lr-AdamW/pg10.00042392669549680005 n2(- _wandbruntime7314  _runtime731498'(O YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated296872206497792 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed64766045 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current249 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 trainer/global_step16449 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated289819383554508 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 _step1313 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14474412336128 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated296872206497792 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated282397794161664 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated47156021 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated297928966014976 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 283000892 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated283062920598528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 _runtime 7314.03174391 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated282397794161664 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 % _timestamp1.7858026012080467e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 111922068 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current251 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14474324906496 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 111921817 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed47155772 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed296872102028800 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14474412336128 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed283062895694848 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed282397777122304 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14438901370552'(O dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated14866045416448 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 109281929 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14865910547968 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed289819280712024 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 109281927 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 173718963 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 173709307 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed282397777122304 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 trainer/global_step16449 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated289819383554508 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 _step1313 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14474412336128 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated296872206497792 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated282397794161664 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated47156021 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated297928966014976 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 283000892 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated283062920598528 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 _runtime 7314.03174391 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated282397794161664 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 % _timestamp1.7858026012080467e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 111922068 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current251 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14474324906496 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 111921817 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed47155772 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed296872102028800 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14474412336128 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed283062895694848 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed282397777122304 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14438901370552 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated296872206497792 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed64766045 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current249 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 J8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _runtime7314.396185259 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 109288572 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 173729500 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14866812432384 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 _step1314 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 109288570 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed47158566 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current259 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14439778843774 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated47158825 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated296890252887552 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 109288572 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated14475291962880 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 173719830 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated297947075009024 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed275397119513792 trainer/global_step16449 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 109288570 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed14475204526080 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed64769982 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14439864647010 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 111928548 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed283080102811648 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current261 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated289837001200162 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed282414943885312 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 283008400 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated14475291962880 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 % _timestamp1.7858026015728517e+09 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 283008400 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 283018072 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 173719830 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 283018072 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 173729500 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated64769984 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated282414960924672 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed297946915244032 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated275397136553152C&(M bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated14866947293696 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated14475291962880 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 173719830 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated297947075009024 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed275397119513792 trainer/global_step16449 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 109288570 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed14475204526080 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed64769982 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14439864647010 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 111928548 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed283080102811648 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current261 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated289837001200162 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed282414943885312 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 283008400 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated14475291962880 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 % _timestamp1.7858026015728517e+09 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 283008400 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 283018072 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 173719830 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 283018072 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 173729500 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated64769984 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated282414960924672 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed297946915244032 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated275397136553152 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _runtime7314.396185259 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 109288572 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 173729500 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14866812432384 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 _step1314 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 109288570 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed47158566 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current259 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14439778843774 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated47158825 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated296890252887552 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 109288572 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 174246965 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 _runtime7336.056677244 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated47299988 $ _timestamp1.785802623230306e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated290700280634016 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated14911292877824 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated297774540783616 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed283256115272704 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed283923251534848 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed64962895 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14482875637184 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 283851386 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 109614077 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current230 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14482789834060 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 109614079 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated276217404996832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed14518321041920 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed297774436314624 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated283256132312064 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed47299758 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed290700177791532 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 112262653 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated298834569316352 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed297774436314624 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated283256132312064 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed298834417932800 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated283923276438528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 112262885 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14518408471552 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed276217387957472 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1-:T(O _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14518408471552 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed276217387957472 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 174246965 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 _runtime7336.056677244 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated47299988 $ _timestamp1.785802623230306e+09 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated290700280634016 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated14911292877824 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated297774540783616 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed283256115272704 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed283923251534848 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed64962895 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14482875637184 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 283851386 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 109614077 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current230 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14482789834060 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 109614079 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated276217404996832 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed14518321041920 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed297774436314624 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated283256132312064 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed47299758 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed290700177791532 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 112262653 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated298834569316352 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed297774436314624 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated283256132312064 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed298834417932800 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated283923276438528 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 112262885 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 Z2(- _wandbruntime7336  _runtime7336g&(M H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed283273282035712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14519200661504 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated297792587173376 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed14519200661504 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 174247832 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14912066562048 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated14519288098304 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 283878224 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed283940458651648 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14483753110518 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 174257502 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed283273282035712 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 109620722 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 109620720 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 112269654 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated290717898279670 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 174257502 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated47302820 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current266 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated14912201423360 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current268 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed298852525213696 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 283878224 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated298852684978688 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 112269386 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed283940458651648 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14483753110518 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 174257502 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed283273282035712 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 109620722 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 109620720 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 112269654 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated290717898279670 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 174257502 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated47302820 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current266 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated14912201423360 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current268 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed298852525213696 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 283878224 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated298852684978688 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 112269386 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated297792587173376 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed14519200661504 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 174247832 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14912066562048 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated14519288098304 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 283878224 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 2(- _wandbruntime7336  _runtime7336 _( 0loss/FRt2/assignment_loss2.0419208347797393 /loss/FRt2/detection_loss2.4922777380328625 /loss/FBt2/detection_loss0.8834717248938978 trainer/global_step16499 _runtime7336.428674687 0loss/FRt1/assignment_loss2.4769814220475563 /loss/SRqqt1/assignment_loss0.7011200295249 2loss/SRqqt2/assignment_loss0.9711289726106859 4loss/FBt1/assignment_loss2.2351739570325435e-08 1loss/SRqqt1/detection_loss0.9507608494038383 &loss/total_loss15.182123739321533 loss/FBt2/assignment_loss0 /loss/FBt1/detection_loss1.0867333322530612 epoch0 $ _timestamp1.785802623605775e+09 _step1319 /loss/FRt1/detection_loss2.5196921130021415 1loss/SRqqt2/detection_loss1.0580367004211113U͸( _runtime7336.428674687 0loss/FRt1/assignment_loss2.4769814220475563 /loss/SRqqt1/assignment_loss0.7011200295249 2loss/SRqqt2/assignment_loss0.9711289726106859 4loss/FBt1/assignment_loss2.2351739570325435e-08 1loss/SRqqt1/detection_loss0.9507608494038383 &loss/total_loss15.182123739321533 loss/FBt2/assignment_loss0 /loss/FBt1/detection_loss1.0867333322530612 epoch0 $ _timestamp1.785802623605775e+09 _step1319 /loss/FRt1/detection_loss2.5196921130021415 1loss/SRqqt2/detection_loss1.0580367004211113 0loss/FRt2/assignment_loss2.0419208347797393 /loss/FRt2/detection_loss2.4922777380328625 /loss/FBt2/detection_loss0.8834717248938978 trainer/global_step16499 H((: U gpu.0.gpu37.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts106.473' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent35.491 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0Vb((:  V disk.nvme0n1.in 4.64453125! disk.nvme0n1.out 303.80859375 disk.drbd1356.in0 disk.rbd1.out 29.521484375 disk.drbd1828.out0 disk.drbd1550.in0 disk.drbd1285.out0 disk.drbd1226.in0 network.recv831366 disk.drbd2118.out0 disk.drbd1285.in0 disk.drbd2078.in0 disk.drbd1584.out0 disk.drbd1552.out0 disk.drbd1938.in0 disk.sda.in1.59375 disk.drbd1828.in0 disk.drbd1289.in0 disk.sda1.out0 disk.drbd1510.in0 disk.drbd1356.out0 disk.drbd1226.out0* proc.memory.percent17.182016372680664$ disk./.usageGB578.3529968261719 disk.rbd0.in0 disk.sda.out 200.75390625 disk.drbd2078.out0 disk.drbd1965.out0 disk.drbd2041.out0 disk.drbd1552.in0 disk.dm-0.in 4.64453125$ memory_percent41.26151204109192) disk./.usagePercent32.34452723242178 disk.drbd1817.out0 disk.drbd1673.out0 disk.rbd0.out0 disk.drbd1289.out0 proc.cpu.threads29 network.sent 1.5230148e+07 disk.drbd1584.in0 disk.drbd1931.out0 cpu13.725115528659202 disk.drbd2118.in0 disk.sda2.in1.59375 disk.rbd1.in0 disk.sda1.in0 disk.sda2.out 200.75390625 disk.drbd2111.in0 disk.drbd1931.in0 disk.dm-0.out 303.80859375* proc.memory.availableMB38494.85546875 disk.drbd1817.in0 disk.drbd1510.out0 disk.drbd2041.in0 disk.drbd1965.in0 disk.drbd1673.in0 disk.drbd1550.out0! proc.memory.rssMB 11260.40625 disk.drbd2111.out0 disk.drbd1938.out0. 2(- _wandbruntime7357  _runtime7357( & lr-AdamW/pg10.0004265039141453306 & lr-AdamW/pg20.0004265039141453306 trainer/global_step16549 % _timestamp1.7858026451339216e+09 _runtime7357.956918151 _step1320e5( % _timestamp1.7858026451339216e+09 _runtime7357.956918151 _step1320 & lr-AdamW/pg10.0004265039141453306 & lr-AdamW/pg20.0004265039141453306 trainer/global_step16549 }G2(- _wandbruntime7357  _runtime7357ISO(O M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 174765311 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated298676875069440 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed298676770600448 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 174765311 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 284721196 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated284114470462464 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14562404606976 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 112602901 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current252 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14526678297568 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 $ _timestamp1.785802645134892e+09 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 109946227 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed47443156 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14526764100692 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed284114453423104 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed299739936540672 _step1321 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed65159745 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed298676770600448 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated277054413612832 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 284711538 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated47443408 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated298676875069440 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current254 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed284783607374848 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed291581074871040 WIDeviceStatsMonitor.on_train_batch_start/jYreserved_bytes.small_pool.current 253755392 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 109946229 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed277054396573472 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated284114470462464 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated14956461937152 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 109946229 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated291581177713524 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 109946227 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14956329165824 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 112603155 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated284783632278528 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 _runtime7357.961657717 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 174774967 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed284114453423104 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14562404606976 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated65159747 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated299740094215680 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed14562317177344 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 174774967 g'(O YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14562317177344 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 trainer/global_step16549 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed14562317177344 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 174774967 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 174765311 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated298676875069440 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed298676770600448 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 174765311 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 284721196 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated284114470462464 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14562404606976 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 112602901 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current252 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14526678297568 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 $ _timestamp1.785802645134892e+09 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 109946227 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed47443156 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14526764100692 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed284114453423104 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed299739936540672 _step1321 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed65159745 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed298676770600448 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated277054413612832 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 284711538 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated47443408 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated298676875069440 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current254 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed284783607374848 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed291581074871040 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 109946229 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed277054396573472 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated284114470462464 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated14956461937152 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 109946229 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated291581177713524 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 109946227 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed14956329165824 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 112603155 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated284783632278528 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 _runtime7357.961657717 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 174774967 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed284114453423104 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14562404606976 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated65159747 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated299740094215680 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 JDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated284800839395328 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 174785504 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated14563284233728 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 284738376 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14957233501696 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _step1322 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated277071153785152 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated298694921459200 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated298694921459200 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated299758203564032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed298694816983040 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 _runtime7358.353476947 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated14563284233728 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed14563196796928 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14527555770790 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed47445986 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated284131637225472 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed284131620186112 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 284728704 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 112609912 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current244 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 174775834 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated65163684 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed299758047993344 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated291598795359178 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 174775834 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated284131637225472 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 284738376 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14527641574026 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 $ _timestamp1.785802645530099e+09 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 109952872 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 112609668 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated47446228 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 109952870 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed277071136745792 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed65163682 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14563196796928 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 109952870 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current242 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed284131620186112 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 284728704 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 109952872 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538n<(M cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated284800839395328 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 174785504 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated14563284233728 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 284738376 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed14957233501696 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _step1322 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated277071153785152 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated298694921459200 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated298694921459200 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated299758203564032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed298694816983040 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 _runtime7358.353476947 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated14563284233728 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed14563196796928 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14527555770790 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed47445986 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated284131637225472 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed284131620186112 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 284728704 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 112609912 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current244 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 174775834 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated65163684 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed299758047993344 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated291598795359178 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 174775834 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated284131637225472 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 284738376 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14527641574026 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 $ _timestamp1.785802645530099e+09 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 109952872 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 112609668 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated47446228 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 109952870 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed277071136745792 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed65163682 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14563196796928 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 109952870 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current242 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed284131620186112 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 284728704 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 109952872 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed291598692516582 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 %YU2(- _wandbruntime7358  _runtime7358+( 2loss/SRqqt2/assignment_loss0.7357534079637844 loss/FBt1/assignment_loss0 /loss/FRt1/detection_loss2.4512281589919613 epoch0 0loss/FRt2/assignment_loss1.8778840967371493 /loss/FBt2/detection_loss0.8006959810705954 % _timestamp1.7858026455317862e+09 _runtime7358.354748782 2loss/SRqqt1/assignment_loss0.6815575252794013 1loss/SRqqt1/detection_loss1.1555721311523472 1loss/SRqqt2/detection_loss1.1286825066988146 /loss/FBt1/detection_loss0.6221913213117255 trainer/global_step16549 0loss/FRt1/assignment_loss2.4442190442766463 3loss/FBt2/assignment_loss6.274172386311923e-09 /loss/FRt2/detection_loss2.3680976165717915 %loss/total_loss14.26588179632839 _step1323[.n( 0loss/FRt1/assignment_loss2.4442190442766463 3loss/FBt2/assignment_loss6.274172386311923e-09 /loss/FRt2/detection_loss2.3680976165717915 %loss/total_loss14.26588179632839 _step1323 2loss/SRqqt2/assignment_loss0.7357534079637844 loss/FBt1/assignment_loss0 /loss/FRt1/detection_loss2.4512281589919613 epoch0 0loss/FRt2/assignment_loss1.8778840967371493 /loss/FBt2/detection_loss0.8006959810705954 % _timestamp1.7858026455317862e+09 _runtime7358.354748782 2loss/SRqqt1/assignment_loss0.6815575252794013 1loss/SRqqt1/detection_loss1.1555721311523472 1loss/SRqqt2/detection_loss1.1286825066988146 /loss/FBt1/detection_loss0.6221913213117255 trainer/global_step16549 10_((:  U disk.drbd1226.out0 disk.drbd2118.out0 disk.drbd1356.out0 disk.drbd1817.out0 disk.drbd1931.in0 disk.dm-0.in 4.64453125 disk.drbd2041.out0 disk.drbd1289.out0 disk.dm-0.out 304.01171875 disk.nvme0n1.in 4.64453125$ memory_percent41.26046299934387 disk.drbd2078.out0 disk.drbd1584.out0* proc.memory.availableMB38495.54296875 disk.drbd1673.in0 disk.drbd1552.in0 disk.drbd1289.in0 disk.rbd0.in0! disk.nvme0n1.out 304.01171875 disk.drbd1356.in0 disk.drbd2111.in0) disk./.usagePercent32.34452829911098$ disk./.usageGB578.3530158996582 disk.drbd1510.in0 disk.drbd1584.in0 disk.drbd2118.in0 network.recv831432 disk.drbd1817.in0 disk.rbd1.out 29.5537109375 disk.drbd1965.in0 disk.drbd1965.out0 disk.drbd2041.in0 disk.rbd0.out0 disk.sda.out 201.234375 disk.drbd1285.in0 disk.sda2.in1.59375 disk.rbd1.in0 disk.drbd1673.out0 disk.drbd1550.in0 disk.sda1.out0 disk.drbd2078.in0 disk.sda2.out 201.234375 disk.drbd1285.out0 disk.drbd1931.out0 disk.drbd1828.in0! proc.memory.rssMB 11260.40625 disk.drbd1510.out0* proc.memory.percent17.182016372680664 disk.drbd2111.out0 disk.sda1.in0 proc.cpu.threads29 disk.sda.in1.59375 network.sent 1.5230214e+07 disk.drbd1828.out0 disk.drbd1938.in0 disk.drbd1226.in0 disk.drbd1552.out0 cpu13.724915005791097 disk.drbd1938.out0 disk.drbd1550.out0((: V gpu.0.gpu24.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts120.326' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent40.108666666666664 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors042(- _wandbruntime7379  _runtime7379'( trainer/global_step16599 $ _timestamp1.785802667115917e+09 _runtime7379.938895788 _step1324 ' lr-AdamW/pg10.00042779252346959593 ' lr-AdamW/pg20.00042779252346959593&{( _runtime7379.938895788 _step1324 ' lr-AdamW/pg10.00042779252346959593 ' lr-AdamW/pg20.00042779252346959593 trainer/global_step16599 $ _timestamp1.785802667115917e+09 i02(- _wandbruntime7379  _runtime73791|'(O WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated285643988118528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 110278379 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current273 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14570566761076 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 285571690 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated277891422228832 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 112944155 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated292462074793032 trainer/global_step16599 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 110278377 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 175293313 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed300645519678464 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14606400742400 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated65356597 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 % _timestamp1.7858026671169171e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed14606313312768 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed299579104886272 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed284972791573504 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed292461971950548 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated300645677353472 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 175302969 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated299579209355264 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14606400742400 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 175302969 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current275 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 112943880 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed65356595 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated299579209355264 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed15001556463616 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed299579104886272 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated15001689234944 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251K{'(O JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current275 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 112943880 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed65356595 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated299579209355264 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed15001556463616 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed299579104886272 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated15001689234944 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated285643988118528 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 110278379 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current273 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14570566761076 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 285571690 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated277891422228832 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 112944155 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated292462074793032 trainer/global_step16599 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 110278377 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 175293313 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed300645519678464 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14606400742400 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated65356597 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 % _timestamp1.7858026671169171e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed14606313312768 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed299579104886272 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed284972791573504 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed292461971950548 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated300645677353472 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 175302969 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated299579209355264 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14606400742400 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 175302969 r6%((: ࣉU gpu.0.gpu24.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts98.771' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent32.92366666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0bc((:  ДU disk.drbd2041.out0 disk.rbd0.in0 disk.drbd1584.out0* proc.memory.percent17.182016372680664 disk.drbd2078.in0 disk.drbd1828.out0 disk.drbd1938.in0 disk.drbd2118.out0 disk.drbd2111.out0$ disk./.usageGB578.3529968261719 disk.dm-0.in 4.64453125 disk.drbd1552.out0 disk.sda.out 201.8828125 disk.drbd1510.in0 disk.sda2.in1.59375 disk.rbd1.in0 disk.drbd1550.in0 disk.rbd0.out0 disk.sda1.out0 disk.drbd1584.in0! proc.memory.rssMB 11260.40625) disk./.usagePercent32.34452723242178 disk.drbd2111.in0 disk.drbd1828.in0 disk.drbd1285.in0 disk.nvme0n1.in 4.64453125 disk.drbd1356.in0 disk.drbd1673.out0 disk.drbd1931.in0 disk.drbd1289.out0 cpu13.724545771939802 disk.drbd1938.out0 disk.sda.in1.59375 disk.drbd1285.out0 disk.drbd1931.out0 disk.drbd1552.in0$ memory_percent41.26197695732117* proc.memory.availableMB38494.55078125 disk.drbd1510.out0 disk.drbd1226.out0 disk.drbd1356.out0 disk.drbd1289.in0 proc.cpu.threads29 disk.drbd2118.in0" disk.nvme0n1.out 304.337890625 disk.drbd1817.in0 disk.drbd1817.out0 disk.drbd1965.in0 disk.sda2.out 201.8828125 disk.drbd1673.in0 network.sent 1.5230364e+07 disk.drbd1226.in0 disk.drbd1965.out0 disk.drbd2041.in0 disk.drbd1550.out0 network.recv831582 disk.sda1.in0 disk.dm-0.out 304.337890625 disk.drbd2078.out0 disk.rbd1.out 29.6474609375Z>2(- _wandbruntime7380  _runtime7380DY9_ (M L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 175313506 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated292479692438686 _step1326 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed65360532 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated284989975375872 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed277908145361792 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 110285020 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed300663630998016 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed14607192932352 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14607192932352 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15002593430528 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserve 88d_bytes.small_pool.current 253755392 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed292479589596090 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 175303836 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated14607280369152 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 175303836 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated65360534 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 285588856 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 175313506 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 % _timestamp1.7858026674993916e+09 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated14607280369152 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 285598528 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current264 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated277908162401152 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 110285022 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14571444234298 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 112950663 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 285598528 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated47590393 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated299597255745024 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 285588856 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed284989958336512 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated299597255745024 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 _runtime 7380.32273424 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 110285022 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 112950927 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 110285020 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 trainer/global_step16599 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated285661195235328 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14571530037534l*v&(M XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed300663630998016 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed14607192932352 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14607192932352 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15002593430528 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed292479589596090 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 175303836 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated14607280369152 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 175303836 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated65360534 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 285588856 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 175313506 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 % _timestamp1.7858026674993916e+09 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated14607280369152 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 285598528 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current264 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated277908162401152 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 110285022 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14571444234298 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 112950663 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 285598528 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated47590393 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated299597255745024 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 285588856 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed284989958336512 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated299597255745024 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 _runtime 7380.32273424 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 110285022 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 112950927 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 110285020 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 trainer/global_step16599 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated285661195235328 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14571530037534 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 175313506 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated292479692438686 _step1326 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed65360532 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated284989975375872 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed277908145361792 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 110285020 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 +2(- _wandbruntime7380  _runtime7380; ( 2loss/FBt2/assignment_loss7.69092098967268e-09 &loss/total_loss14.842306767183928 trainer/global_step16599 _step1327 2loss/SRqqt2/assignment_loss0.7761112593628952 /loss/FRt1/detection_loss2.3713422000945843 /loss/SRqqt1/detection_loss1.42943631936796 /loss/FBt2/detection_loss0.9076732418710186 2loss/SRqqt1/assignment_loss0.6615233229434428 1loss/SRqqt2/detection_loss1.1002211031511802 % _timestamp1.7858026675013735e+09 0loss/FRt1/assignment_loss2.1906105120157457 /loss/FRt2/assignment_loss1.771549936336807 /loss/FRt2/detection_loss2.2229329362700256 .loss/FBt1/detection_loss1.410905918145242 epoch0 _runtime7380.324185973 3loss/FBt1/assignment_loss9.934106870446158e-09ól) _step1327 2loss/SRqqt2/assignment_loss0.7761112593628952 /loss/FRt1/detection_loss2.3713422000945843 /loss/SRqqt1/detection_loss1.42943631936796 /loss/FBt2/detection_loss0.9076732418710186 2loss/SRqqt1/assignment_loss0.6615233229434428 1loss/SRqqt2/detection_loss1.1002211031511802 % _timestamp1.7858026675013735e+09 0loss/FRt1/assignment_loss2.1906105120157457 /loss/FRt2/assignment_loss1.771549936336807 /loss/FRt2/detection_loss2.2229329362700256 .loss/FBt1/detection_loss1.410905918145242 epoch0 _runtime7380.324185973 3loss/FBt1/assignment_loss9.934106870446158e-09 2loss/FBt2/assignment_loss7.69092098967268e-09 &loss/total_loss14.842306767183928 trainer/global_step16599 f)(: ߜU gpu.0.gpu29.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts119.409' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent39.803 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0^,b)(:  U" disk.nvme0n1.out 304.533203125 disk.sda2.in1.59375 disk.drbd1552.out0 disk.drbd1828.out0 disk.sda1.in0 disk.drbd1510.out0 disk.drbd1965.in0 disk.drbd1285.in0 disk.drbd1356.in0 disk.drbd2111.out0 disk.drbd1828.in0 disk.sda.in1.59375 disk.drbd1817.in0 disk.rbd1.in0 disk.dm-0.in 4.64453125 disk.drbd2118.out0 disk.drbd1938.out0 disk.drbd1510.in0 disk.drbd1584.out0 disk.sda2.out 202.296875 disk.drbd1289.in0 cpu13.724160687217134 disk.nvme0n1.in 4.64453125 network.sent 1.5277994e+07 disk.drbd2111.in0 disk.rbd1.out 29.7099609375 disk.drbd1552.in0 disk.drbd1550.out0 disk.drbd1226.out0$ memory_percent41.26165509223938* proc.memory.availableMB38494.76171875* disk./.usagePercent32.344445524028735$ disk./.usageGB578.3515357971191 disk.drbd2078.out0 disk.drbd2118.in0 disk.drbd2041.out0 network.recv833196 disk.drbd1673.out0 disk.drbd1931.out0 disk.drbd2078.in0* proc.memory.percent17.182016372680664 disk.drbd1965.out0 disk.drbd1289.out0 disk.dm-0.out 304.533203125 disk.drbd1584.in0 disk.drbd1931.in0 disk.drbd1938.in0 disk.rbd0.in0 disk.drbd1226.in0! proc.memory.rssMB 11260.40625 disk.drbd1285.out0 disk.drbd1356.out0 disk.drbd1817.out0 disk.drbd2041.in0 disk.drbd1550.in0 disk.rbd0.out0 disk.sda1.out0 disk.drbd1673.in0 disk.sda.out 202.296875 proc.cpu.threads29rL2)- _wandbruntime7402  _runtime7402z) _runtime7402.300703148 _step1328 & lr-AdamW/pg10.0004290811327938612 & lr-AdamW/pg20.0004290811327938612 trainer/global_step16649 $ _timestamp1.785802689476103e+09) & lr-AdamW/pg20.0004290811327938612 trainer/global_step16649 $ _timestamp1.785802689476103e+09 _runtime7402.300703148 _step1328 & lr-AdamW/pg10.0004290811327938612 c2)- _wandbruntime7402  _runtime7402B~R')O NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 _runtime7402.300866064 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated300481543641088 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 286441500 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated47731750 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 175830971 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed293342869030056 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 % _timestamp1.7858026894768238e+09 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated65553447 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14614541027708 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 286431842 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated278728430844832 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed300481439172096 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated285831146763264 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 175821315 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 175821315 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14650309448192 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current246 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed14650309448192 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 _step1329 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed300481439172096 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current248 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed285831129723904 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14650396877824 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated286504343958528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated15046934951424 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated47731750 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 175830971 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed293342869030056 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 % _timestamp1.7858026894768238e+09 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated65553447 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14614541027708 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 286431842 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated278728430844832 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed300481439172096 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated285831146763264 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 175821315 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 175821315 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14650309448192 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current246 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed14650309448192 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 _step1329 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 (2)- _wandbruntime7402  _runtime7402̤a9&)M D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14615418501042 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 % _timestamp1.7858026898551865e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 175841508 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed285848296486912 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 175841508 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current242 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed300499485554688 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated65557384 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed65557382 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 trainer/global_step16649 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed293360486675598 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated300499590030848 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _runtime7402.678564125 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 110617172 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed301569235397120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated285848313526272 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 286458680 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated293360589518194 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed15047709225472 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current244 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated47734562 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed14651189067776 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14615332697806 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 110617170 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 175831838 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 110617170 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated14651276504576 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 113291702 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated14651276504576 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 175831838 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated286521551075328 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated285848313526272 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15047839892480 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 286449008 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed286521526171648 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated301569390967808 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0QΌ4&)M WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed285848296486912 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14651189067776 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated293360589518194 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed15047709225472 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current244 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated47734562 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed14651189067776 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14615332697806 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 110617170 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 175831838 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 110617170 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated14651276504576 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 113291702 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated14651276504576 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 175831838 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated286521551075328 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated285848313526272 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15047839892480 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 286449008 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed286521526171648 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated301569390967808 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14615418501042 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 % _timestamp1.7858026898551865e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 175841508 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed285848296486912 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 175841508 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current242 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed300499485554688 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated65557384 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed65557382 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 trainer/global_step16649 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed293360486675598 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated300499590030848 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _runtime7402.678564125 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 110617172 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed301569235397120 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated285848313526272 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 286458680 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 EDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed301383773457920 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed15092052190720 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 176349317 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 110942679 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14694305583616 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current235 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed279565422421472 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed47875206 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated286689484913664 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 110942679 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed286689467874304 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated301383877926912 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 176349317 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated65750297 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated47875441 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 % _timestamp1.7858027114755452e+09 _runtime7424.298786262 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed302456727085568 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14658343688092 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 176358973 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 113625501 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated287364699798528 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14694393013248 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14694393013248 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated294223868952048 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 trainer/global_step16699 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed301383773457920 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated301383877926912 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated286689484913664 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed286689467874304 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 _step1333 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 287301652 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0+_')O `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed15092052190720 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 176349317 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 110942679 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14694305583616 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current235 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed279565422421472 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed47875206 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated286689484913664 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 110942679 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed286689467874304 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated301383877926912 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 176349317 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated65750297 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated47875441 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 % _timestamp1.7858027114755452e+09 _runtime7424.298786262 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed302456727085568 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14658343688092 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 176358973 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 113625501 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated287364699798528 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14694393013248 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14694393013248 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated294223868952048 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 trainer/global_step16699 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed301383773457920 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated301383877926912 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated286689484913664 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed286689467874304 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 _step1333 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 287301652 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed14694305583616 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated15092184962048 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14658429491216 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 110942677 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 176358973 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current237 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated302456884760576 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated279565439460832 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 113625738 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed301383773457920 6?2)- _wandbruntime7424  _runtime7424&)M FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 287318832 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current229 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 287309160 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed279582162593792 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated279582179633152 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14659221161314 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed302474839330816 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 287318832 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated65754234 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated287381906915328 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14695185203200 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15093087986176 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated294241486597702 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 113632295 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed65754232 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 trainer/global_step16699 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 110949322 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 110949320 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated302474994901504 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed14695185203200 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed287381882011648 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 _step1334 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed301401819840512 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 % _timestamp1.7858027118489838e+09 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 110949322 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated301401924316672 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 176359840 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed286706634637312 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121)M L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated47878292 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated14695272640000 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed294241383755106 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated65754234 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated287381906915328 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14695185203200 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15093087986176 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated294241486597702 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 113632295 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed65754232 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 trainer/global_step16699 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 110949322 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 110949320 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated302474994901504 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed14695185203200 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed287381882011648 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 _step1334 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed301401819840512 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 % _timestamp1.7858027118489838e+09 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 110949322 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated301401924316672 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 176359840 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed286706634637312 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 287318832 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current229 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 287309160 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed279582162593792 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated279582179633152 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14659221161314 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed302474839330816 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 287318832 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 EDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed65947145 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 111274827 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14702317954724 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 111274829 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated287547823064064 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed280402431037472 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed287547806024704 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 288161804 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current233 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated295104766031556 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed295104663189072 _runtime7446.217284964 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed302286107743744 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14738389148672 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 176886975 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed15137261422080 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 176877319 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14738389148672 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 113966912 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated288225055638528 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed302286107743744 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 trainer/global_step16749 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated15137394193408 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated287547823064064 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current235 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14702232151600 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 % _timestamp1.7858027333940108e+09 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated303362449831936 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated302286212212736 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated302286212212736 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121x )O RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed280402431037472 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed287547806024704 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 288161804 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current233 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated295104766031556 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed295104663189072 _runtime7446.217284964 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed302286107743744 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14738389148672 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 176886975 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed15137261422080 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 176877319 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14738389148672 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 113966912 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated288225055638528 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed302286107743744 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 trainer/global_step16749 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated15137394193408 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated287547823064064 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current235 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14702232151600 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 % _timestamp1.7858027333940108e+09 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated303362449831936 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated302286212212736 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated302286212212736 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed48019532 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated65947147 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed287547806024704 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 113966677 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated48019765 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 288152146 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 176877319 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed303362292156928 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed14738301719040 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated280402448076832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 SDDeviceStatsMonitor.on_trai_^n_batch_start/active_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 176886975 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed288225030734848 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 111274827 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 _step1337 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed65947145 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 111274827 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14702317954724 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 111274829 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated287547823064064 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 p 2)- _wandbruntime7446  _runtime7446mi&)M _step1338 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14703109624822 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 113973509 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated303380569074176 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 176897512 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed287564972787712 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 111281472 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed288242237851648 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed15138177748992 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 288169312 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 _runtime7446.588036111 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed65951082 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated287564989827072 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed280419171209792 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14703195428058 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current255 trainer/global_step16749 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 176897512 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 113973764 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 288178984 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated288242262755328 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated280419188249152 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 111281472 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 288178984 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated302304258602496 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated287564989827072 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current253 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated295122383677210 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 111281470 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15138306318848 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated14739268775424 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14739181338624 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 176897512 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 113973764 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 288178984 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated288242262755328 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated280419188249152 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 111281472 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 288178984 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated302304258602496 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated287564989827072 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current253 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated295122383677210 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 111281470 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15138306318848 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated14739268775424 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14739181338624 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14703109624822 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 113973509 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated303380569074176 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 176897512 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed287564972787712 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 111281472 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed288242237851648 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed15138177748992 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 288169312 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 _runtime7446.588036111 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed65951082 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated287564989827072 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed280419171209792 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14703195428058 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current255 ;l2)- _wandbruntime7446  _runtime7446 $a ) 0loss/FRt2/assignment_loss2.2424326224907025 /loss/FRt2/detection_loss2.5814153828935043 1loss/SRqqt1/detection_loss1.0651761158172723 _runtime7446.589154917 _step1339 0loss/FRt1/assignment_loss2.2946501342873824 2loss/SRqqt2/assignment_loss0.7804785145312179 3loss/FBt1/assignment_loss3.703906778745087e-06 3loss/FBt2/assignment_loss1.440443305374591e-07 /loss/FBt1/detection_loss1.1733681677641081 /loss/FBt2/detection_loss1.2671063461651404 &loss/total_loss15.690054321168954 trainer/global_step16749 2loss/SRqqt1/assignment_loss0.7793657632846333 /loss/FRt1/detection_loss2.3023161178748857 1loss/SRqqt2/detection_loss1.2037413081089998 epoch0 $ _timestamp1.785802733766326e+09mgl$) 0loss/FRt2/assignment_loss2.2424326224907025 /loss/FRt2/detection_loss2.5814153828935043 1loss/SRqqt1/detection_loss1.0651761158172723 _runtime7446.589154917 _step1339 0loss/FRt1/assignment_loss2.2946501342873824 2loss/SRqqt2/assignment_loss0.7804785145312179 3loss/FBt1/assignment_loss3.703906778745087e-06 3loss/FBt2/assignment_loss1.440443305374591e-07 /loss/FBt1/detection_loss1.1733681677641081 /loss/FBt2/detection_loss1.2671063461651404 &loss/total_loss15.690054321168954 trainer/global_step16749 2loss/SRqqt1/assignment_loss0.7793657632846333 /loss/FRt1/detection_loss2.3023161178748857 1loss/SRqqt2/detection_loss1.2037413081089998 epoch0 $ _timestamp1.785802733766326e+09 |9)(: ۋU gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts129.299' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent43.09966666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0t c)(:  U disk.drbd1828.in0 disk.drbd1289.out0 disk.sda1.out0 disk.drbd1510.in0 disk.sda2.out 204.2109375 disk.drbd1673.out0 disk.drbd1931.out0 disk.rbd0.in0# disk.nvme0n1.out305.4599609375 network.sent 1.5397333e+07 disk.sda2.in1.59375 disk.drbd1938.in0 disk.rbd0.out0 disk.drbd2118.out0 disk.drbd1550.out0 disk.sda1.in0 disk.drbd1817.in0 disk.drbd1550.in0 disk.drbd1226.in0 disk.drbd1356.in0 disk.drbd1285.in0 disk.drbd1510.out0 disk.rbd1.in0 proc.cpu.threads29 network.recv837271 disk.drbd2041.out0 disk.drbd1552.in0 disk.drbd1817.out0 disk.drbd1226.out0 disk.drbd1584.in0 disk.drbd2118.in0 disk.drbd1965.in0* proc.memory.availableMB38493.75390625) disk./.usagePercent32.34450675198906$ disk./.usageGB578.3526306152344 cpu13.722871176125668 disk.nvme0n1.in 4.64453125 disk.drbd2111.in0 disk.sda.out 204.2109375 disk.dm-0.out305.4599609375 disk.drbd1828.out0 disk.drbd1289.in0 disk.drbd2111.out0 disk.drbd1552.out0 disk.dm-0.in 4.64453125 disk.drbd1356.out0 disk.drbd1673.in0 disk.drbd1931.in0 disk.drbd1584.out0 disk.rbd1.out 29.8984375 disk.drbd1965.out0% memory_percent41.263192892074585* proc.memory.percent17.182016372680664 disk.drbd2041.in0 disk.drbd1285.out0 disk.drbd2078.out0! proc.memory.rssMB 11260.40625 disk.sda.in1.59375 disk.drbd1938.out0 disk.drbd2078.in08u@2)- _wandbruntime7468  _runtime7468+) trainer/global_step16799 $ _timestamp1.785802755488252e+09 _runtime7468.311210969 _step1340 & lr-AdamW/pg10.0004329469607666572 & lr-AdamW/pg20.0004329469607666572) _runtime7468.311210969 _step1340 & lr-AdamW/pg10.0004329469607666572 & lr-AdamW/pg20.0004329469607666572 trainer/global_step16799 $ _timestamp1.785802755488252e+09 02)- _wandbruntime7468  _runtime7468gc)O ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed304267854932480 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 _step1341 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed303188442029568 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated304268016801792 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 % _timestamp1.7858027554891863e+09 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 114306656 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current257 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14782385284096 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed66143995 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed289085386574848 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated48162659 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14782297854464 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 289021956 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 111606977 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 177405321 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated15182605323264 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current255 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 111606977 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 177414977 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated288406161214464 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 trainer/global_step16799 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated281239456692832 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated288406161214464 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed303188442029568 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocatiQ/ons.current0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated295985663111064 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 111606979 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed288406144175104 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 289012298 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 177405321 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 111606979 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14746206418232 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14746120615108 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed288406144175104 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14782385284096 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated289085411478528 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed303188442029568 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated304268016801792 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 % _timestamp1.7858027554891863e+09 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 114306656 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current257 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14782385284096 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed66143995 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed289085386574848 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated48162659 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14782297854464 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 289021956 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 111606977 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 177405321 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated15182605323264 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current255 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 111606977 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 177414977 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated288406161214464 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 trainer/global_step16799 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated281239456692832 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated288406161214464 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed303188442029568 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated295985663111064 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 111606979 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed288406144175104 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 289012298 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 177405321 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 111606979 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14746206418232 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14746120615108 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed288406144175104 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14782385284096 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated289085411478528 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 JDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 289039136 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 289039136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed296003177914122 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 289029464 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current260 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed48165262 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed281256179825792 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated303206592888320 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14747083891566 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14783177474048 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated14783264910848 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 111613620 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 trainer/global_step16799 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated304286124059136 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated288423327977472 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 114313454 _runtime7468.686811343 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 111613620 _step1342 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 111613622 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed304285964294144 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 177415844 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated288423327977472 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 % _timestamp1.7858027558634546e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated14783264910848 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 177425514 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated281256196865152 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed288423310938112 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15183505463808 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated296003280756718 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated289102618595328 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed289102593691648 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed15183370602496 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 114313194 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 177425514 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0Ǘ")M GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current260 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed48165262 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed281256179825792 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated303206592888320 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14747083891566 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14783177474048 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated14783264910848 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 111613620 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 trainer/global_step16799 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated304286124059136 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated288423327977472 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 114313454 _runtime7468.686811343 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 111613620 _step1342 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 111613622 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed304285964294144 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 177415844 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated288423327977472 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 % _timestamp1.7858027558634546e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated14783264910848 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 177425514 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated281256196865152 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed288423310938112 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15183505463808 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated296003280756718 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated289102618595328 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed289102593691648 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed15183370602496 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 114313194 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 177425514 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed66147932 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed14783177474048 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 289039136 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 289039136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed296003177914122 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 289029464 2)- _wandbruntime7468  _runtime7468) 0loss/FRt1/assignment_loss2.4280709449363793 .loss/FRt1/detection_loss2.281264264136553 epoch0 0loss/FRt2/assignment_loss2.2579211277621134 0loss/SRqqt2/detection_loss1.308397903572768 % _timestamp1.7858027558652365e+09 1loss/SRqqt1/assignment_loss0.855913331358822 loss/FBt1/assignment_loss0 .loss/FRt2/detection_loss2.567893961178405 1loss/SRqqt1/detection_loss1.4995935607846056 &loss/total_loss16.822107731501703 _step1343 2loss/SRqqt2/assignment_loss0.7066138512331795 3loss/FBt2/assignment_loss8.064153287618329e-08 /loss/FBt1/detection_loss1.7207104319240898 /loss/FBt2/detection_loss1.1957282739732524 trainer/global_step16799 _runtime7468.6880632977a) 1loss/SRqqt1/assignment_loss0.855913331358822 loss/FBt1/assignment_loss0 .loss/FRt2/detection_loss2.567893961178405 1loss/SRqqt1/detection_loss1.4995935607846056 &loss/total_loss16.822107731501703 _step1343 2loss/SRqqt2/assignment_loss0.7066138512331795 3loss/FBt2/assignment_loss8.064153287618329e-08 /loss/FBt1/detection_loss1.7207104319240898 /loss/FBt2/detection_loss1.1957282739732524 trainer/global_step16799 _runtime7468.688063297 0loss/FRt1/assignment_loss2.4280709449363793 .loss/FRt1/detection_loss2.281264264136553 epoch0 0loss/FRt2/assignment_loss2.2579211277621134 0loss/SRqqt2/detection_loss1.308397903572768 % _timestamp1.7858027558652365e+09 f)(:  U* proc.memory.percent17.182016372680664 proc.cpu.threads29 disk.sda.out 204.55859375 disk.dm-0.in 4.64453125 disk.drbd1584.out0 disk.drbd2118.in0 disk.sda2.out 204.55859375 disk.drbd2111.in0 disk.drbd1552.in0 disk.drbd1285.in0 disk.drbd2078.in0 disk.rbd1.out 29.8984375 disk.sda.in1.59375 disk.drbd1510.out0 disk.drbd1226.in0 disk.drbd2041.in0 disk.drbd2041.out0 disk.drbd2118.out0 disk.drbd1356.out0 disk.drbd2111.out0 network.sent 1.546615e+07 disk.drbd1289.in0 disk.drbd1938.in0 disk.rbd0.in0 disk.rbd0.out0 disk.drbd1817.out0$ disk./.usageGB578.3526306152344 disk.drbd1550.in0 disk.drbd1510.in0 cpu13.722594180413692) disk./.usagePercent32.34450675198906! disk.dm-0.out305.75634765625 disk.nvme0n1.in 4.64453125 disk.drbd1356.in0 disk.drbd1673.out0 disk.drbd1828.in0 disk.drbd1226.out0 disk.sda2.in1.59375 disk.rbd1.in0 disk.drbd1828.out0 disk.drbd1817.in0 disk.drbd1965.out0 disk.drbd1289.out0 disk.sda1.in0 disk.sda1.out0 disk.drbd1285.out0 disk.drbd1673.in0 disk.drbd1931.in0 disk.drbd1938.out0 network.recv839248 disk.drbd1931.out0 disk.drbd1584.in0$ disk.nvme0n1.out305.75634765625% memory_percent41.262441873550415* proc.memory.availableMB38494.24609375! proc.memory.rssMB 11260.40625 disk.drbd1965.in0 disk.drbd1552.out0 disk.drbd1550.out0 disk.drbd2078.out0)(: 誇U gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts132.539' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent44.17966666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0H)(: ձU gpu.0.gpu38.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts127.044' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent42.348 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0-[)(:  U disk.nvme0n1.out 306.0546875 cpu13.72216733094893 network.sent 1.5466216e+07 disk.drbd1550.in0# memory_percent41.2625253200531* proc.memory.percent17.182016372680664$ disk./.usageGB578.3526496887207 disk.drbd1938.out0 disk.rbd0.out0 disk.drbd2078.out0 disk.drbd1584.out0 disk.drbd1673.in0 disk.drbd1931.in0 disk.drbd1828.out0 network.recv839314 disk.drbd1510.in0 disk.drbd1289.out0 disk.sda1.out0 disk.drbd2118.in0 disk.drbd1817.out0 disk.drbd1965.out0 disk.drbd1938.in0 disk.drbd1510.out0) disk./.usagePercent32.34450781867827 disk.sda.out 204.796875 disk.drbd1285.in0 disk.drbd1226.out0 disk.rbd1.in0 disk.rbd1.out 29.9306640625* proc.memory.availableMB38494.19140625 disk.sda.in1.59375 disk.drbd1356.in0 disk.drbd2111.in0 disk.drbd2041.in0 disk.drbd1828.in0 disk.rbd0.in0 disk.drbd2041.out0 disk.drbd1552.in0 disk.drbd1226.in0 disk.drbd1584.in0 proc.cpu.threads29 disk.sda1.in0 disk.dm-0.out 306.0546875 disk.drbd2078.in0 disk.drbd2118.out0 disk.drbd1965.in0 disk.drbd1931.out0! proc.memory.rssMB 11260.40625 disk.sda2.in1.59375 disk.drbd1552.out0 disk.drbd1289.in0 disk.dm-0.in 4.64453125 disk.nvme0n1.in 4.64453125 disk.sda2.out 204.796875 disk.drbd1285.out0 disk.drbd1356.out0 disk.drbd1817.in0 disk.drbd1673.out0 disk.drbd2111.out0 disk.drbd1550.out0$ G2)- _wandbruntime7490  _runtime7490(f) ' lr-AdamW/pg10.00043423557009092243 ' lr-AdamW/pg20.00043423557009092243 trainer/global_step16849 % _timestamp1.7858027777725854e+09 _runtime7490.595558326 _step1344v7 ) trainer/global_step16849 % _timestamp1.7858027777725854e+09 _runtime7490.595558326 _step1344 ' lr-AdamW/pg10.00043423557009092243 ' lr-AdamW/pg20.00043423557009092243 Wփ2)- _wandbruntime7490  _runtime7490sl')O F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 114648151 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 111939129 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed304090776315392 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14826293989888 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14790094881740 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current230 _step1345 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 177942979 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 111939127 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated289264499364864 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed289264482325504 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 289872450 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed304090776315392 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated289945767318528 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed15227658812416 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 trainer/global_step16849 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed282076448269472 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 114647919 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed48307074 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed296866457348088 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated305173560999424 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14826381419520 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated289264499364864 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _runtime7490.596779183 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated48307304 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed289945742414848 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated304090880784384 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed289264482325504 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14826381419520 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed305173401227264 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 177933323 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360p6')O UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated289264499364864 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 _runtime7490.596779183 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated48307304 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed289945742414848 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated304090880784384 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed289264482325504 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14826381419520 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed305173401227264 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 177933323 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 114648151 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 111939129 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed304090776315392 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14826293989888 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14790094881740 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current230 _step1345 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 177942979 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 111939127 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated289264499364864 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed289264482325504 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 289872450 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed304090776315392 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated289945767318528 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed15227658812416 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 trainer/global_step16849 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed282076448269472 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 114647919 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed48307074 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed296866457348088 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated305173560999424 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14826381419520 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 %2)- _wandbruntime7490  _runtime7490W)M PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 111945772 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed48309954 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed305191535132160 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 111945770 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated296884177836226 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14827173609472 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 111945772 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 177943846 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated14827261046272 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 114654987 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 114654736 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated289962974435328 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 289899288 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed289962949531648 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed289281649088512 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 124375552 BTB&)M JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 289889616 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 177953516 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 289899288 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed289962949531648 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed289281649088512 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 124375552 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 114654987 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 114654736 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated289962974435328 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 PDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 178461325 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 114990457 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated290122837515264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed48452514 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated297747457270080 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated306079063997952 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated15272940839424 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14870290125312 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed297747354427596 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 112271277 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14833897542124 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 290732602 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed15272805970944 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed290806098254848 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated290122837515264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated48452760 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current248 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 _runtime7512.312797578 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed14870290125312 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 178470981 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 178461325 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated304993215070208 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed304993110601216 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated66537697 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 trainer/global_step16899 _step1349 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated282913473924832 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 # _timestamp1.78580279948918e+09 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed304993110601216 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14870377554944 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current246 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 114990209 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 178461325 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 114990457 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated290122837515264 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed48452514 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated297747457270080 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated306079063997952 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated15272940839424 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14870290125312 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed297747354427596 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 112271277 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14833897542124 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 290732602 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed15272805970944 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed290806098254848 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated290122837515264 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated48452760 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current248 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 _runtime7512.312797578 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed14870290125312 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 178470981 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 178461325 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated304993215070208 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed304993110601216 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated66537697 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 trainer/global_step16899 _step1349 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated282913473924832 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 # _timestamp1.78580279948918e+09 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed304993110601216 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14870377554944 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current246 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 114990209 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 114997032 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated282930214097152 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed282930197057792 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 112277920 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 112277922 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 178481518 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed290139987238912 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated306097172197376 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 290759440 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 # _timestamp1.78580279986522e+09 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed14871169744896 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated66541634 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 178471848 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15273841922048 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _runtime7512.688575512 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 112277920 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed305011156983808 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed297764972073138 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed15273711255040 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed306097016626688 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14871169744896 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 112277922 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated305011261459968 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 290759440 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14834775015346 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 290749768 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current249 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated290140004278272 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated48455649 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14834860818582 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed48455400 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated00\&)M QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed48455400 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated297765074915734 trainer/global_step16899 _step1350 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 290749768 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 114997283 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 178481518 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 114997032 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated282930214097152 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed282930197057792 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 112277920 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 112277922 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 178481518 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed290139987238912 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated306097172197376 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 290759440 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 # _timestamp1.78580279986522e+09 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed14871169744896 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated66541634 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 178471848 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15273841922048 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _runtime7512.688575512 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 112277920 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed305011156983808 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed297764972073138 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed15273711255040 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed306097016626688 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 155570688 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14871169744896 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 112277922 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated305011261459968 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 290759440 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14834775015346 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 290749768 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current249 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated290140004278272 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated48455649 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14834860818582 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 2)- _wandbruntime7512  _runtime7512kNq) 1loss/SRqqt2/detection_loss1.0270831523270443 .loss/FBt2/detection_loss0.909599361523729 epoch0 trainer/global_step16899 2loss/SRqqt1/assignment_loss0.7380308559031359 0loss/SRqqt1/detection_loss1.093046232712056 /loss/FBt1/detection_loss0.9272157372190402 $ _timestamp1.785802799866966e+09 /loss/FRt2/assignment_loss2.004622433758989 &loss/total_loss14.564048766947662 _runtime7512.689809726 3loss/FBt1/assignment_loss5.501965461007785e-08 .loss/FRt1/detection_loss2.484842038431833 /loss/FRt2/detection_loss2.2713056088342696 _step1351 0loss/FRt1/assignment_loss2.3159433873586877 2loss/SRqqt2/assignment_loss0.7923599038592235 loss/FBt2/assignment_loss0C) 2loss/SRqqt2/assignment_loss0.7923599038592235 loss/FBt2/assignment_loss0 1loss/SRqqt2/detection_loss1.0270831523270443 .loss/FBt2/detection_loss0.909599361523729 epoch0 trainer/global_step16899 2loss/SRqqt1/assignment_loss0.7380308559031359 0loss/SRqqt1/detection_loss1.093046232712056 /loss/FBt1/detection_loss0.9272157372190402 $ _timestamp1.785802799866966e+09 /loss/FRt2/assignment_loss2.004622433758989 &loss/total_loss14.564048766947662 _runtime7512.689809726 3loss/FBt1/assignment_loss5.501965461007785e-08 .loss/FRt1/detection_loss2.484842038431833 /loss/FRt2/detection_loss2.2713056088342696 _step1351 0loss/FRt1/assignment_loss2.3159433873586877 ][)(: ӆU gpu.0.gpu37.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp42.0 gpu.0.powerWatts128.388' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent42.796 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0(_)(:  U$ disk./.usageGB578.3526954650879 disk.drbd2118.in0 disk.drbd1356.out0 disk.drbd1817.out0! proc.memory.rssMB 11260.40625$ disk.nvme0n1.out306.91748046875 disk.drbd2041.in0 disk.drbd1289.in0 proc.cpu.threads29) disk./.usagePercent32.34451037873235 disk.drbd1965.out0 network.recv843020 disk.drbd2118.out0 disk.drbd1673.out0 disk.drbd1938.out0 network.sent 1.5561963e+07 disk.drbd2078.out0 disk.drbd2111.out0 disk.sda2.in1.59375 disk.drbd1817.in0 disk.drbd1552.out0 disk.rbd0.out0 disk.sda.out 205.84375 disk.nvme0n1.in 4.64453125 disk.rbd1.out 30.1103515625 disk.drbd1931.out0 disk.rbd0.in0& proc.memory.availableMB 38493.3125 disk.drbd1285.out0 disk.dm-0.in 4.64453125 disk.sda2.out 205.84375 disk.drbd2041.out0* proc.memory.percent17.182016372680664 disk.drbd1938.in0 disk.drbd1226.out0 disk.drbd1285.in0 disk.drbd1584.out0 disk.drbd1673.in0 disk.drbd1931.in0 disk.drbd1510.in0 disk.drbd1226.in0 disk.drbd2078.in0 disk.rbd1.in0 disk.drbd1356.in0 disk.drbd1965.in0 disk.drbd1552.in0 disk.drbd1550.out0 disk.sda.in1.59375 disk.drbd1828.in0 disk.drbd1550.in0$ memory_percent41.26386642456055 disk.drbd1584.in0 disk.drbd1828.out0 disk.drbd1289.out0 disk.sda1.in0 disk.sda1.out0 disk.drbd1510.out0! disk.dm-0.out306.91748046875 disk.drbd2111.in0 cpu13.7215038572628538@ )(: U gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp41.0 gpu.0.powerWatts127.666' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent42.55533333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0=`g)(:  U) disk./.usagePercent32.34451037873235 disk.drbd1584.out0 disk.drbd1673.out0 disk.drbd1550.in0$ memory_percent41.26381278038025 disk.drbd1289.out0 disk.drbd1226.in0 disk.drbd1817.in0 disk.drbd1817.out0* proc.memory.percent17.182016372680664 disk.drbd1285.in0 disk.drbd2078.in0 disk.drbd2078.out0 disk.drbd2118.in0 disk.sda2.out 206.1328125 disk.drbd1552.out0 network.sent 1.5562029e+07 disk.rbd0.out0 disk.dm-0.in 4.64453125 disk.sda2.in1.59375! proc.memory.rssMB 11260.40625! disk.dm-0.out307.55908203125 disk.drbd2111.out0 proc.cpu.threads29 disk.rbd0.in0 disk.sda1.in0$ disk.nvme0n1.out307.55908203125 disk.drbd1356.in0 disk.drbd1931.in0 cpu13.721230635155472 disk.sda1.out0 disk.drbd1510.out0 disk.drbd1356.out0 disk.drbd2111.in0 disk.rbd1.out 30.1728515625 disk.drbd2041.in0 disk.drbd1828.out0 disk.drbd1550.out0 disk.drbd1584.in0 network.recv843086$ disk./.usageGB578.3526954650879 disk.drbd1828.in0 disk.drbd1938.in0 disk.drbd1510.in0 disk.drbd1931.out0 disk.drbd1552.in0 disk.sda.in1.59375 disk.drbd1285.out0 disk.drbd1226.out0 disk.rbd1.in0 disk.drbd1965.out0 disk.drbd1673.in0 disk.nvme0n1.in 4.64453125 disk.drbd2041.out0 disk.sda.out 206.1328125 disk.drbd1289.in0 disk.drbd1938.out0* proc.memory.availableMB38493.34765625 disk.drbd2118.out0 disk.drbd1965.in0UW2)- _wandbruntime7534  _runtime7534(a{) ' lr-AdamW/pg20.00043681278873945305 trainer/global_step16949 % _timestamp1.7858028217642736e+09 _ry:Nuntime 7534.5872602 _step1352 ' lr-AdamW/pg10.00043681278873945305=.) _step1352 ' lr-AdamW/pg10.00043681278873945305 ' lr-AdamW/pg20.00043681278873945305 trainer/global_step16949 % _timestamp1.7858028217642736e+09 _runtime 7534.5872602 ê2)- _wandbruntime7534  _runtime7534>@%')O LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 291602412 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current242 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 112603429 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 178998983 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated298628354349588 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed14914286260736 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed14914286260736 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14877871808756 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated48597562 % _timestamp1.7858028217652936e+09 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14914373690368 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed66734545 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated66734547 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed291666454094848 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current240 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14877786005632 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 trainer/global_step16949 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 178989327 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14914373690368 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed298628251507104 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 _runtime7534.588610408 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 112603427 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated305895549356032 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 291592754 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated290981175665664 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 178998983 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated290981175665664 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed305895444887040 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed290981158626304 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed305895444887040 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 112603429 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 178989327 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated283750482540832 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated305895549356032 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 115332109 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14877786005632 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 trainer/global_step16949 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 178989327 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated14914373690368 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed298628251507104 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 _runtime7534.588610408 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 112603427 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated305895549356032 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 291592754 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated290981175665664 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 178998983 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated290981175665664 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed305895444887040 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed290981158626304 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed305895444887040 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 112603429 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ɴ32)- _wandbruntime7534  _runtime7534 &)M A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 179009520 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated66738484 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed15318885250560 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 291609920 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15319024306176 _step1354 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14878749282090 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 112610072 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated14915253317120 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 115338887 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 % _timestamp1.7858028221348176e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 115338650 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated290998342428672 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current235 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated290998342428672 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 163959296 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 291619592 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed307002546462208 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated305913595745792 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 291619592 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed291683661211648 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 178999850 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated291683686115328 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated305913595745792 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _runtime7534.958163901 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14878663478854 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 291609920 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 112610072 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 179009520 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14915165880320 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed48600168 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed290998325389312 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current237 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated298645971995242 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130wO͇)M FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 112610072 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated14915253317120 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 115338887 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 % _timestamp1.7858028221348176e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 115338650 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated290998342428672 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current235 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated290998342428672 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 163959296 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 291619592 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed307002546462208 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated305913595745792 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 291619592 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed291683661211648 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 178999850 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated291683686115328 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated305913595745792 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 _runtime7534.958163901 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14878663478854 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 291609920 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 112610072 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 179009520 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14915165880320 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed48600168 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed290998325389312 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current237 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated298645971995242 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 179009520 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated66738484 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed15318885250560 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 291609920 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15319024306176 _step1354 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14878749282090 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14958369825792 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed306797779172864 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed48741434 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 112935579 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed284587474117472 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed66931395 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed15363175209984 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 _runtime7556.756115151 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed299509148586612 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed292526809934848 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated66931397 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 179526985 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated15363305884160 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 _step1357 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated307890140722688 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 % _timestamp1.7858028439328835e+09 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 292462564 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed307889985144832 trainer/global_step16999 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14921674469140 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 292452906 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated291839513816064 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated48741688 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated306797883641856 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 115672829 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed14958282396160 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 112935577 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136<K')O VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed299509148586612 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed292526809934848 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated66931397 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 179526985 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated15363305884160 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 _step1357 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated307890140722688 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 % _timestamp1.7858028439328835e+09 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 292462564 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed307889985144832 trainer/global_step16999 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed14921674469140 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 292452906 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated291839513816064 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated48741688 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated306797883641856 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 115672829 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed14958282396160 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 112935577 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated306797883641856 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed291839496776704 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated284587491156832 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current256 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated14958369825792 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed306797779172864 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed48741434 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 112935579 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed284587474117472 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed66931395 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed15363175209984 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 _runtime7556.756115151 z\)- _wandbruntimet7557  _runtime7557vh&)M ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated291856680579072 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated66935334 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed48744380 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed299526766232154 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed14959162015744 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 _step1358 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated14959249452544 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated306815930031616 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 179537522 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated307908249503744 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 trainer/global_step16999 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 115679712 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated48744629 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 112942220 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 179527852 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 112942222 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 112942220 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current249 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 292479744 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _runtime7557.130328073 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 292479744 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated292544041955328 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated284604231329152 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 112942222 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 % _timestamp1.7858028443069923e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated14959249452544 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed291856663539712 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 115679963 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed15364078978560 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed66935332 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14959162015744 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 292470072 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 179527852 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated299526869074750 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 179537522 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 292479744 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _runtime7557.130328073 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 292479744 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated292544041955328 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated284604231329152 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 112942222 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 % _timestamp1.7858028443069923e+09 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated14959249452544 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed291856663539712 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 115679963 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed15364078978560 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed66935332 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed14959162015744 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 292470072 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 179527852 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated299526869074750 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 179537522 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated307908249503744 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 trainer/global_step16999 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 115679712 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated48744629 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 112942220 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 179527852 !2)- _wandbruntime7557  _runtime7557) %loss/total_loss17.16707248102492 /loss/FRt1/assignment_loss2.638914473851522 2loss/SRqqt2/assignment_loss0.8296864111562846 /loss/FRt2/detection_loss2.7133866259029933 1loss/SRqqt1/detection_loss1.0664103699352157 /loss/FBt1/detection_loss1.7697663809522055 /loss/FBt2/detection_loss1.3934754574360946 1loss/SRqqt1/assignment_loss0.964083655555914 loss/FBt2/assignment_loss0 .loss/FRt1/detection_loss2.381357855598132 0loss/FRt2/assignment_loss2.2664157552378517 1loss/SRqqt2/detection_loss1.1435754879481264 trainer/global_step16999 % _timestamp1.7858028443089705e+09 _runtime7557.131800832 _step1359 3loss/FBt1/assignment_loss7.450580152834618e-09 epoch0jr) 1loss/SRqqt1/assignment_loss0.964083655555914 loss/FBt2/assignment_loss0 .loss/FRt1/detection_loss2.381357855598132 0loss/FRt2/assignment_loss2.2664157552378517 1loss/SRqqt2/detection_loss1.1435754879481264 trainer/global_step16999 % _timestamp1.7858028443089705e+09 _runtime7557.131800832 _step1359 3loss/FBt1/assignment_loss7.450580152834618e-09 epoch0 %loss/total_loss17.16707248102492 /loss/FRt1/assignment_loss2.638914473851522 2loss/SRqqt2/assignment_loss0.8296864111562846 /loss/FRt2/detection_loss2.7133866259029933 1loss/SRqqt1/detection_loss1.0664103699352157 /loss/FBt1/detection_loss1.7697663809522055 /loss/FBt2/detection_loss1.3934754574360946 @7)(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp40.0 gpu.0.powerWatts97.229' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent32.409666666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0M th)(:  U disk.drbd1552.out0 disk.drbd1550.in0 network.sent 1.5652168e+07) disk./.usagePercent32.34451037873235 disk.rbd0.in0 disk.drbd1356.out0 disk.drbd1817.out0 disk.drbd1673.in0 disk.drbd1828.in0 disk.drbd1226.in0 disk.drbd2118.out0 disk.nvme0n1.in 4.64453125 disk.drbd2111.out0 cpu13.720588949368418 disk.drbd1931.out0! proc.memory.rssMB 11260.40625 proc.cpu.threads29 disk.sda1.out0 disk.drbd2118.in0 network.recv845968 disk.drbd1931.in0 disk.rbd0.out0 disk.drbd1226.out0$ disk.nvme0n1.out308.14208984375 disk.drbd1817.in0 disk.sda.out 206.97265625 disk.drbd1285.in0 disk.drbd1356.in0 disk.rbd1.out 30.298828125 disk.drbd1828.out0 disk.dm-0.in 4.64453125 disk.drbd1938.in0! disk.dm-0.out308.14208984375 disk.drbd1552.in0 disk.drbd2078.in0 disk.sda2.out 206.97265625 disk.drbd2041.in0 disk.drbd1285.out0 disk.drbd2111.in0 disk.drbd1289.out0 disk.rbd1.in0 disk.drbd1289.in0 disk.sda.in1.59375) proc.memory.availableMB 38492.9296875* proc.memory.percent17.182016372680664 disk.drbd1550.out0 disk.drbd2078.out0$ disk./.usageGB578.3526954650879 disk.drbd1965.in0 disk.drbd1510.in0 disk.drbd1584.out0 disk.drbd1673.out0 disk.sda1.in0 disk.drbd1510.out0 disk.sda2.in1.59375 disk.drbd1584.in0% memory_percent41.264450550079346 disk.drbd1938.out0 disk.drbd1965.out0 disk.drbd2041.out0O)(: ȌU gpu.0.gpu30.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp40.0 gpu.0.powerWatts106.865' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent35.62166666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0f)(:  ٱU disk.drbd1584.out0 disk.drbd1289.in0 disk.drbd1285.out0 disk.drbd2118.out0 disk.drbd2041.out0$ memory_percent41.26378893852234 network.recv846076$ disk./.usageGB578.3526382446289 disk.rbd0.out0 proc.cpu.threads29 disk.drbd2078.in0 disk.drbd1356.in0 disk.sda1.out0 disk.drbd1510.in0 disk.drbd1226.in0) disk./.usagePercent32.34450717866474 disk.drbd2118.in0 disk.drbd2041.in0 disk.drbd1552.in0! proc.memory.rssMB 11260.40625 disk.drbd2111.in0 disk.drbd1285.in0 disk.drbd1965.out0 disk.drbd1673.in0 disk.drbd1550.in0 disk.dm-0.in 4.64453125! disk.dm-0.out308.31787109375 disk.drbd1931.out0 cpu13.720287917786582 disk.rbd1.out 30.298828125 disk.drbd1938.out0 disk.sda.out 207.5546875* proc.memory.percent17.182016372680664 disk.drbd1510.out0 network.sent 1.5652276e+07 disk.nvme0n1.in 4.64453125 disk.drbd1356.out0 disk.rbd1.in0 disk.drbd1828.in0 disk.drbd1550.out0 disk.sda.in1.59375 disk.drbd1584.in0$ disk.nvme0n1.out308.31787109375 disk.drbd2111.out0 disk.drbd1828.out0 disk.drbd2078.out0 disk.sda2.in1.59375 disk.drbd1817.in0 disk.rbd0.in0 disk.drbd1226.out0* proc.memory.availableMB38493.36328125 disk.sda2.out 207.5546875 disk.drbd1673.out0 disk.sda1.in0 disk.drbd1931.in0 disk.drbd1965.in0 disk.drbd1552.out0 disk.drbd1289.out0 disk.drbd1938.in0 disk.drbd1817.out0յ~2)- _wandbruntime7578  _runtime7578]) & lr-AdamW/pg10.0004393900073879837 & lr-AdamW/pg20.0004393900073879837 trainer/global_step17049 % _timestamp1.7858028659628294e+09 _runtime7578.785838697 _step1360M!) & lr-AdamW/pg10.0004393900073879837 & lr-AdamW/pg20.0004393900073879837 trainer/global_step17049 % _timestamp1.7858028659628294e+09 _runtime7578.785838697 _step1360 I,2)- _wandbruntime7578  _runtime7578Bw)O QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated48885802 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 180045331 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed15002278531584 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 180054987 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 113267729 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 293322716 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated293387190678528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 _step1361 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 180054987 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed307700113458688 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 113267729 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14965648735772 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 116014049 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed307700113458688 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current255 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed293387165774848 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated307700217927680 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated292697851966464 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed308795478685696 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated285424499772832 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated15002365961216 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated300390148508604 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated15002365961216 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated308795638457856 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed300390045666120 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 113267727 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 VHDeviceStatsMonitor.on_train_batch_start/2oinactive_split_bytes.all.current 159772160 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 % _timestamp1.7858028659638474e+09 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 293313058 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed292697834927104 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated15408447779328 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed15408312910848 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 180054987 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed307700113458688 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 113267729 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated14965648735772 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 116014049 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed307700113458688 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current255 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed293387165774848 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated307700217927680 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated292697851966464 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed308795478685696 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated285424499772832 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated15002365961216 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated300390148508604 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated15002365961216 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated308795638457856 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed300390045666120 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 113267727 7Z2*- _wandbruntime7579  _runtime7579 ҆&*M T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated307718264317440 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 293330224 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current256 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated300407766154258 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed307718159841280 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed15003158151168 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14966440405870 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated15003245587968 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current258 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated67132184 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated292715018729472 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 293339896 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed67132182 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed15409220038144 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed300407663311662 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 113274372 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 180065524 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed308813592929792 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 180065524 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated307718264317440 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated308813746403328 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated285441239945152 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 293339896 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _step1362 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed285441222905792 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 293330224 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 113274372 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed292715001690112 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed15003158151168 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 % _timestamp1.7858028663390462e+09 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 180055854 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed292715001690112 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 116020698 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed48888516 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14966526209106 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 113274370 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated292715018729472 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 trainer/global_step17049 _runtime7579.162392686wa"*M H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated285441239945152 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 293339896 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _step1362 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed285441222905792 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 293330224 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 113274372 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed292715001690112 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed15003158151168 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 % _timestamp1.7858028663390462e+09 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 180055854 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed292715001690112 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 116020698 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed48888516 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated14966526209106 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 113274370 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated292715018729472 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 trainer/global_step17049 _runtime7579.162392686 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated307718264317440 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 293330224 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current256 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated300407766154258 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed307718159841280 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed15003158151168 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed14966440405870 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated15003245587968 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current258 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated67132184 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated292715018729472 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 293339896 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed67132182 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed15409220038144 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed300407663311662 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 113274372 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 180065524 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed308813592929792 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 VGDeviceStatsMonitor.on_train_batch_end/resrCUerved_bytes.large_pool.current 2768240640 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 180065524 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated307718264317440 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated308813746403328 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 u>2*- _wandbruntime7579  _runtime7579* 0loss/FRt1/assignment_loss2.4332407558901927 0loss/FRt2/assignment_loss1.9479610120548922 epoch0 trainer/global_step17049 % _timestamp1.7858028663407013e+09 _step1363 3loss/FBt2/assignment_loss3.973642748178463e-09 &loss/total_loss14.703828812019413 2loss/SRqqt1/assignment_loss0.7019656676634137 .loss/FRt1/detection_loss2.457988342165612 .loss/FRt2/detection_loss2.205965209708494 .loss/FBt1/detection_loss1.246571437089837 /loss/FBt2/detection_loss0.7653366683982312 2loss/SRqqt2/assignment_loss0.7203199182566423 loss/FBt1/assignment_loss0 0loss/SRqqt1/detection_loss1.147270678706021 1loss/SRqqt2/detection_loss1.0772091181124333 _runtime7579.163544243~J* 0loss/FRt1/assignment_loss2.4332407558901927 0loss/FRt2/assignment_loss1.9479610120548922 epoch0 trainer/global_step17049 % _timestamp1.7858028663407013e+09 _step1363 3loss/FBt2/assignment_loss3.973642748178463e-09 &loss/total_loss14.703828812019413 2loss/SRqqt1/assignment_loss0.7019656676634137 .loss/FRt1/detection_loss2.457988342165612 .loss/FRt2/detection_loss2.205965209708494 .loss/FBt1/detection_loss1.246571437089837 /loss/FBt2/detection_loss0.7653366683982312 2loss/SRqqt2/assignment_loss0.7203199182566423 loss/FBt1/assignment_loss0 0loss/SRqqt1/detection_loss1.147270678706021 1loss/SRqqt2/detection_loss1.0772091181124333 _runtime7579.163544243 Ĭj\*(: U gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp40.0 gpu.0.powerWatts125.593' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent41.864333333333335 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0e{`*(:  U disk.drbd1226.out0 disk.drbd1584.in0 disk.rbd1.out 30.361328125 disk.drbd1550.in0 disk.drbd1289.in0 network.sent 1.5697242e+07 disk.sda2.out208.0625 disk.drbd1817.in0 disk.drbd1828.in0 disk.sda1.out0) disk./.usagePercent32.34451037873235 disk.drbd1673.in0 disk.sda.out208.0625 disk.drbd2078.out0 disk.drbd2111.out0 disk.drbd1673.out0 disk.drbd2078.in0 disk.drbd1226.in0 disk.drbd1552.out0! proc.memory.rssMB 11260.40625 disk.drbd2118.in0 disk.drbd1510.in0 disk.drbd1584.out0 disk.rbd0.out0 disk.dm-0.in 4.64453125$ disk.nvme0n1.out308.64013671875 disk.drbd1356.out0 disk.sda2.in1.59375 disk.drbd1289.out0 disk.drbd1510.out0 disk.drbd1552.in0 disk.rbd0.in0$ disk./.usageGB578.3526954650879 disk.drbd1285.out0 disk.drbd2111.in0 disk.rbd1.in0 disk.drbd1931.in0 disk.drbd2041.in0 disk.drbd1938.in0 network.recv847430 disk.nvme0n1.in 4.64453125 disk.drbd1356.in0* proc.memory.availableMB38493.05078125 disk.drbd1817.out0 disk.drbd1965.in0 disk.sda.in1.59375 cpu13.720393705431288 proc.cpu.threads29 disk.drbd1285.in0! disk.dm-0.out308.64013671875 disk.drbd1931.out0 disk.drbd2041.out0 disk.drbd1828.out0 disk.drbd1550.out0 disk.drbd1938.out0$ memory_percent41.26426577568054* proc.memory.percent17.182016372680664 disk.drbd2118.out0 disk.drbd1965.out0 disk.sda1.in0Hu2*- _wandbruntime7601  _runtime7601"a* % _timestamp1.7858028885614703e+09 _runtime7601.384453587 _step1364 % lr-AdamW/pg10.000440678616712249 % lr-AdamW/pg20.000440678616712249 trainer/global_step17099.p;* % lr-AdamW/pg10.000440678616712249 % lr-AdamW/pg20.000440678616712249 trainer/global_step17099 % _timestamp1.7858028885614703e+09 _runtime7601.384453587 _step1364 m2*- _wandbruntime7601  _runtime76014w '*O NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed308602447744512 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed67325095 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 113599879 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated15046362096640 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated293556190116864 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed309700985815040 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current238 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated293556190116864 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 113599877 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated308602552213504 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 180582989 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 294173210 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed15046274667008 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed308602447744512 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated15046362096640 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed15453464200192 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed15009451396156 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 _step1365 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 113599879 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated286261508388832 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15046274667008 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated67325097 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated15453586485760 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 294182868 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 180573333 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated309701133004288 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed294247521614848 _runtime7601.385668565 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 113599877 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 116356123 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 116355883 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed301270942745628 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed49030788 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 147189248 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated294247546518528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated15009537199280 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 122285568 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 $ _timestamp1.785802888562452e+09 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 294182868 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 180573333 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated309701133004288 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed294247521614848 _runtime7601.385668565 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 113599877 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 116356123 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 116355883 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed301270942745628 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed49030788 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 147189248 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated294247546518528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated15009537199280 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 122285568 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 $ _timestamp1.785802888562452e+09 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed308602447744512 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed67325095 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 113599879 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated15046362096640 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated293556190116864 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed309700985815040 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current238 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated293556190116864 yz2*- _wandbruntime7601  _runtime7601 J*M H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 294200048 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated308620598603264 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 113606520 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 180593526 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated294264753635328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 141152768 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed286278231521792 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated293573356879872 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15454501060608 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 294200048 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 166056448 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 180583856 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated301288663233766 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated49033883 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed15047154286592 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed15454359907840 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed308620494127104 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current253 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated293573356879872 BDeviceStatsMonitor.on_train_batch_end/inactive_splitwL .all.freed 116362662 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated67329034 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated309719254695936 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 180583856 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 _runtime7601.756398054 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed293573339840512 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 294190376 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed49033630 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 113606522 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed309719088639488 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 _step1366 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated15010414672614 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed15010328869378 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed294264728731648 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 116362917 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 113606520 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 180593526 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated294264753635328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 141152768 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed286278231521792 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated293573356879872 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15454501060608 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 294200048 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 166056448 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 180583856 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated301288663233766 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated49033883 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed15047154286592 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed15454359907840 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed308620494127104 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current253 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated293573356879872 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 116362662 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated67329034 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated309719254695936 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 180583856 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 _runtime7601.756398054 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed293573339840512 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 294190376 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed49033630 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 113606522 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed309719088639488 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 _step1366 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated15010414672614 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed15010328869378 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed294264728731648 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 116362917 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 PDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 181110991 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current267 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed310606541553664 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated295107902358528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated302151942667620 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 181101335 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated67521947 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed67521945 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed15090270802432 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated309504886499328 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed49173696 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed287098499965472 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated49173961 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current265 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed309504782030336 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated15090358232064 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 113932027 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15090270802432 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 113932029 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 181110991 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated15053425662788 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed15053339859664 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 _runtime7623.305374519 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated294414528267264 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed294414511227904 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 113932029 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed294414511227904 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed302151839825136 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated287098517004832 trainer/global_step17149 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 295043020 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed15498664098816 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 % _timestamp1.7858029104821956e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed309504782030336 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 116695641 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed295107877454848 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 181110991 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current267 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed310606541553664 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated295107902358528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated302151942667620 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 181101335 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated67521947 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed67521945 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed15090270802432 MGDeviceStatsMonitor.on_trai2Sn_batch_start/reserved_bytes.small_pool.freed0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated309504886499328 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed49173696 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed287098499965472 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated49173961 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current265 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed309504782030336 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated15090358232064 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 113932027 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15090270802432 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 113932029 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 181110991 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated15053425662788 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed15053339859664 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 _runtime7623.305374519 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated294414528267264 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed294414511227904 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 113932029 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed294414511227904 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed302151839825136 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated287098517004832 trainer/global_step17149 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 295043020 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed15498664098816 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 % _timestamp1.7858029104821956e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed309504782030336 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 116695641 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed295107877454848 JDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 116702375 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 181111858 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated15091237858816 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 trainer/global_step17149 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed302169457470678 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed309522828412928 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed15499569014272 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated309522932889088 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 295060200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 181121528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current256 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated309522932889088 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed15091150422016 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed67525882 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated310624811253760 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated294431695030272 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 _step1370 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 113938670 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 _runtime7623.684910601 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed310624653585920 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated15054303136122 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed294431677990912 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated294431695030272 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated67525884 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated295125109475328 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed287115240137792 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed15091150422016 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 113938672 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated49176749 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 113938672 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed294431677990912 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed295125084571648 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed15054217332886 % _timestamp1.7858029108615668e+09T}&*M E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed287115240137792 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed15091150422016 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 113938672 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated49176749 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 113938672 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed294431677990912 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed295125084571648 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed15054217332886 % _timestamp1.7858029108615668e+09 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 295060200 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 295050528 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 116702375 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 181111858 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated15091237858816 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 trainer/global_step17149 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 132764160 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed302169457470678 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed309522828412928 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed15499569014272 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated309522932889088 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 295060200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 181121528 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current256 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated309522932889088 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed15091150422016 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed67525882 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated310624811253760 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated294431695030272 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 _step1370 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 113938670 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 _runtime7623.684910601 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed310624653585920 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated15054303136122 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed294431677990912 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated294431695030272 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated67525884 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated295125109475328 G 2*- _wandbruntime7623  _runtime7623 e;* 0loss/FRt2/assignment_loss2.2201210468104393 2loss/SRqqt2/assignment_loss0.8388870339820046 .loss/FRt2/detection_loss2.640021413790457 /loss/FBt1/detection_loss0.8756259859073907 loss/FBt1/assignment_loss0 .loss/FRt1/detection_loss2.312272250324815 /loss/FBt2/detection_loss0.6818358711898327 epoch0 $ _timestamp1.785802910863197e+09 _runtime7623.686017809 _step1371 2loss/SRqqt1/assignment_loss0.8335127661244549 3loss/FBt2/assignment_loss2.507496650776312e-07 1loss/SRqqt2/detection_loss1.1719780111174405 %loss/total_loss14.85478158503825 /loss/FRt1/assignment_loss2.321560564931932 1loss/SRqqt1/detection_loss0.9589663901098185 trainer/global_step17149[ * 0loss/FRt2/assignment_loss2.2201210468104393 2loss/SRqqt2/assignment_loss0.8388870339820046 .loss/FRt2/detection_loss2.640021413790457 /loss/FBt1/detection_loss0.8756259859073907 loss/FBt1/assignment_loss0 .loss/FRt1/detection_loss2.312272250324815 /loss/FBt2/detection_loss0.6818358711898327 epoch0 $ _timestamp1.785802910863197e+09 _runtime7623.686017809 _step1371 2loss/SRqqt1/assignment_loss0.8335127661244549 3loss/FBt2/assignment_loss2.507496650776312e-07 1loss/SRqqt2/detection_loss1.1719780111174405 %loss/total_loss14.85478158503825 /loss/FRt1/assignment_loss2.321560564931932 1loss/SRqqt1/detection_loss0.9589663901098185 trainer/global_step17149 U,*(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts124.769' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent41.589666666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0ˊֿi*(:  U disk.drbd1673.out0$ memory_percent41.26397967338562 network.recv849856 disk.drbd1550.out0 disk.drbd1938.in0! disk.dm-0.out309.26806640625 disk.nvme0n1.in 4.64453125 disk.sda2.in1.59375 disk.drbd1817.out0* proc.memory.availableMB38493.23828125 disk.dm-0.in 4.64453125 disk.drbd1226.in0$ disk.nvme0n1.out309.26806640625 disk.drbd1965.out0 disk.drbd1289.in0 disk.drbd1817.in0 disk.rbd1.in0 disk.sda1.out0 disk.drbd1673.in0 disk.drbd2041.out0 cpu13.720745835108444 disk.drbd1510.in0 disk.drbd1965.in0! proc.memory.rssMB 11260.40625) disk./.usagePercent32.34450717866474 disk.rbd0.out0 disk.drbd2078.out0 disk.sda2.out 209.48046875* proc.memory.percent17.182016372680664 proc.cpu.threads29$ disk./.usageGB578.3526382446289 disk.drbd1226.out0 disk.rbd1.out 30.6591796875 disk.drbd2078.in0 disk.drbd1931.in0 disk.drbd1931.out0 disk.drbd1285.in0 disk.drbd2111.out0 disk.drbd2041.in0 disk.drbd1356.in0 disk.drbd1552.out0 disk.drbd1828.in0 disk.sda1.in0 disk.drbd1356.out0 disk.sda.out 209.48046875 disk.drbd1550.in0 disk.drbd1584.out0 disk.drbd2111.in0 disk.drbd1828.out0 disk.sda.in1.59375 disk.drbd1285.out0 disk.drbd1510.out0 disk.drbd1584.in0 disk.drbd2118.out0 disk.drbd1552.in0 network.sent 1.5748079e+07 disk.rbd0.in0 disk.drbd2118.in0 disk.drbd1289.out0 disk.drbd1938.out0CD2*- _wandbruntime7645  _runtime76450Q* % _timestamp1.7858029331014085e+09 _runtime7645.924368511 _step1372 ' lr-AdamW/pg10.00044325583536077956 ' lr-AdamW/pg20.00044325583536077956 trainer/global_step17199?E* _step1372 ' lr-AdamW/pg10.00044325583536077956 ' lr-AdamW/pg20.00044325583536077956 trainer/global_step17199 % _timestamp1.7858029331014085e+09 _runtime7645.924368511 2*- _wandbruntime7645  _runtime7645UN*O UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 181638993 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed311512057544704 _runtime7645.925686741 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 114264179 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 114264177 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated295272866417664 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 295893514 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15134266937856 D>DeviceStatsMonitor.*N!on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated310407220785152 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed295272849378304 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed295272849378304 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated287935525620832 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 181629337 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current246 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed49317183 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed295968233294848 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 295903172 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 181629337 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed67718795 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed310407116316160 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current244 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated295272866417664.'*O JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated295272866417664 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 295893514 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15134266937856 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated310407220785152 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed295272849378304 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed295272849378304 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated287935525620832 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 181629337 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current246 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed49317183 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed295968233294848 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 295903172 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 181629337 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed67718795 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed310407116316160 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current244 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated295272866417664 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 181638993 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed311512057544704 _runtime7645.925686741 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 114264179 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 114264177 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 Jm2*- _wandbruntime7646  _runtime7646&*M OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed15544719449600 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated287952265793152 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 trainer/global_step17199 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed67722732 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated15135233994240 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed303050354550186 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 114270820 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 295910680 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 _runtime7646.313788436 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed15135146557440 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 117042722 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated311530328014848 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed295290016141312 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated49320243 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 114270822 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 114270820 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated67722734 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15544862699520 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 181639860 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated310425267174912 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed295290016141312 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed15135146557440 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated295985465315328 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated303050457392782 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 168153600 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 143249920 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated15098191599630 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed287952248753792 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _step1374 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 295920352 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated295290033180672 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated310425267174912 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current255 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed311530159861248 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed15098105796394 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 181639860 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 295920352 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 117042977 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated295290033180672 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 181649530 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated15135233994240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed49319990 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 114270822 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 295910680 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887*M HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 114270822 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 295910680 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed15544719449600 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated287952265793152 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 trainer/global_step17199 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed67722732 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated15135233994240 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed303050354550186 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 114270820 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 295910680 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 _runtime7646.313788436 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed15135146557440 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 117042722 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated311530328014848 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed295290016141312 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated49320243 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 114270822 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 114270820 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated67722734 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15544862699520 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 181639860 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated310425267174912 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed295290016141312 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed15135146557440 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated295985465315328 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated303050457392782 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 168153600 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 143249920 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated15098191599630 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed287952248753792 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 _step1374 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 295920352 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated295290033180672 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated310425267174912 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current255 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed311530159861248 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed15098105796394 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 181639860 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 295920352 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 117042977 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated295290033180672 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 181649530 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated15135233994240 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed49319990 E 2*- _wandbruntime7646  _runtime7646V6B* 0loss/FRt1/assignment_loss2.1599402908819267 2loss/SRqqt1/assignment_loss0.7755535651101835 1loss/SRqqt2/assignment_loss0.798913695154682 1loss/SRqqt2/detection_loss1.0016168809850656 &loss/total_loss15.332880281879442 trainer/global_step17199 # _timestamp1.78580293349222e+09 _runtime7646.315077395 loss/FBt1/assignment_loss0 _step1375 /loss/FRt2/assignment_loss2.200032032918239 .loss/FRt1/detection_loss2.491728916225663 /loss/FRt2/detection_loss2.6175130291477493 0loss/SRqqt1/detection_loss1.155914233583543 epoch0 4loss/FBt2/assignment_loss2.5096686553488804e-08 /loss/FBt1/detection_loss1.1867354605036478 /loss/FBt2/detection_loss0.94493215227205512* /loss/FRt2/detection_loss2.6175130291477493 0loss/SRqqt1/detection_loss1.155914233583543 epoch0 4loss/FBt2/assignment_loss2.5096686553488804e-08 /loss/FBt1/detection_loss1.1867354605036478 /loss/FBt2/detection_loss0.9449321522720551 0loss/FRt1/assignment_loss2.1599402908819267 2loss/SRqqt1/assignment_loss0.7755535651101835 1loss/SRqqt2/assignment_loss0.798913695154682 1loss/SRqqt2/detection_loss1.0016168809850656 &loss/total_loss15.332880281879442 trainer/global_step17199 # _timestamp1.78580293349222e+09 _runtime7646.315077395 loss/FBt1/assignment_loss0 _step1375 /loss/FRt2/assignment_loss2.200032032918239 .loss/FRt1/detection_loss2.491728916225663 ء *(: ⪒U gpu.0.gpu32.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts98.875' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent32.958333333333336 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0J[-h*(:  U disk.drbd1510.in0 disk.sda.out 209.8515625 proc.cpu.threads29 disk.rbd1.in0 disk.drbd1673.in0 disk.drbd1510.out0 network.recv851476 disk.drbd1673.out0 disk.drbd1584.out0 disk.drbd1552.in0 disk.drbd1584.in0 disk.drbd1817.in0 disk.rbd0.out0 disk.drbd2078.in0 disk.drbd1356.out0 disk.drbd1931.out0 disk.drbd1828.in0 disk.drbd2078.out0% memory_percent41.263777017593384 disk.drbd1285.out0 disk.dm-0.in 4.64453125 disk.drbd1226.in0 network.sent 1.5816734e+07 disk.drbd2118.out0$ disk.nvme0n1.out309.55126953125 disk.drbd1817.out0 disk.sda1.out0 disk.drbd1965.in0 disk.rbd0.in0! proc.memory.rssMB 11260.40625$ disk./.usageGB578.3526954650879 disk.drbd1938.in0 disk.sda2.in1.59375 disk.drbd1285.in0 cpu13.721159538022244 disk.sda2.out 209.8515625 disk.rbd1.out 30.7216796875 disk.drbd1550.in0* proc.memory.percent17.182016372680664) disk./.usagePercent32.34451037873235 disk.drbd2111.out0 disk.nvme0n1.in 4.64453125 disk.drbd1356.in0 disk.drbd1965.out0 disk.drbd1931.in0 disk.drbd1938.out0 disk.drbd1226.out0* proc.memory.availableMB38493.37109375 disk.drbd1289.in0! disk.dm-0.out309.55126953125 disk.drbd2041.in0 disk.drbd2041.out0 disk.sda1.in0 disk.drbd2118.in0 disk.drbd2111.in0 disk.drbd1552.out0 disk.drbd1828.out0 disk.drbd1550.out0 disk.drbd1289.out0 disk.sda.in1.59375*(: U gpu.0.gpu40.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts97.113' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent32.371 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0*j*(:  U network.recv852468 disk.drbd1965.out0! proc.memory.rssMB 11260.40625 disk.drbd2041.in0 disk.drbd1584.out0 disk.drbd2118.in0 disk.drbd2111.in0 disk.drbd1931.out0% memory_percent41.264158487319946! disk.dm-0.out309.76220703125 disk.drbd1965.in0$ disk./.usageGB578.3527145385742 disk.drbd1828.in0 disk.drbd1938.in0 disk.rbd0.out0 disk.sda.in1.59375 disk.drbd2118.out0 disk.sda2.in1.59375 disk.drbd1817.out0* proc.memory.percent17.182016372680664 disk.sda1.in0 disk.drbd2078.in0 disk.drbd1931.in0 cpu13.721393362227403 network.sent 1.5819841e+07 disk.drbd1552.out0 disk.drbd1510.out0 disk.drbd2078.out0* proc.memory.availableMB38493.12109375 proc.cpu.threads29) disk./.usagePercent32.34451144542156 disk.drbd1289.in0 disk.drbd1289.out0 disk.drbd1673.in0 disk.dm-0.in 4.64453125 disk.rbd0.in0 disk.drbd1817.in0 disk.drbd1550.in0 disk.sda.out 210.10546875 disk.drbd1285.out0 disk.sda2.out 210.10546875 disk.nvme0n1.in 4.64453125 disk.drbd1356.in0 disk.drbd1356.out0 disk.rbd1.out 30.7216796875 disk.drbd1828.out0 disk.drbd1285.in0 disk.drbd1550.out0 disk.sda1.out0 disk.drbd1584.in0 disk.drbd1226.out0$ disk.nvme0n1.out309.76220703125 disk.drbd2111.out0 disk.drbd2041.out0 disk.drbd1552.in0 disk.drbd1226.in0 disk.drbd1673.out0 disk.drbd1938.out0 disk.drbd1510.in0 disk.rbd1.in0с+2*- _wandbruntime7668  _runtime7668?* trainer/global_step17249 % _timestamp1.7858029553359153e+09 _runtime7668.158892374 _step1376 ' lr-AdamW/pg10.00044454444468504487 ' lr-AdamW/pg20.00044454444468504487tP* ' lr-AdamW/pg10.00044454444468504487 ' lr-AdamW/pg20.00044454444468504487 trainer/global_step17249 % _timestamp1.7858029553359153e+09 _runtime7668.158892374 _step1376 Sugl2*- _wandbruntime7668  _runtime7668B*'*O \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated15178350502912 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 182157339 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated296828614038528 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 182157339 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated311309555070976 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed296131187528704 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 296763324 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed49460573 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated311309555070976 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15178263073280 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed312417572540416 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated15589111982592 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 114596327 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated67915647 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed15141116786680 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 296753666 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed15178263073280 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current244 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed15588983405568 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 trainer/global_step17249 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated312417726021120 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed296828589134848 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed67915645 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed303913633984152 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed288772517197472 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed311309450601984 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 % _timestamp1.7858029553368785e+09 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed311309450601984 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 117376464 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated49460817 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 114596329 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated296131204568064 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 114596329 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 182166995 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated296131204568064 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed296131187528704 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated15178350502912 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 114596327 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 117376218 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated303913736826636 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392Xn'*O R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed311309450601984 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 % _timestamp1.7858029553368785e+09 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed311309450601984 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 117376464 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated49460817 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 114596329 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated296131204568064 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 114596329 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 182166995 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated296131204568064 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed296131187528704 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated15178350502912 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 114596327 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 117376218 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated303913736826636 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated15178350502912 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 182157339 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated296828614038528 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 182157339 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated311309555070976 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed296131187528704 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 296763324 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed49460573 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated311309555070976 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15178263073280 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed312417572540416 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated15589111982592 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 114596327 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated67915647 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed15141116786680 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 296753666 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed15178263073280 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current244 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed15588983405568 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 trainer/global_step17249 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated312417726021120 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed296828589134848 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed67915645 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed303913633984152 \HDeviceStatsMonitor.on_2Itrain_batch_start/requested_bytes.large_pool.freed288772517197472 v2*- _wandbruntime7668  _runtime7668&*M S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed311327496984576 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 114602970 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated296148371331072 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated296148371331072 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed15179142692864 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 _runtime7668.532801031 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 117383057 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 trainer/global_step17249 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed15179142692864 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current252 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 114602970 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated49463725 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15590017816576 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed67919582 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated15179230129664 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated311327601460736 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 296780504 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed49463475 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed296148354291712 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed296148354291712 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 296770832 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed15589887149568 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 182177532 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated312435838971904 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated296845821155328 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 182167862 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121N>&*M J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed296148354291712 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed296148354291712 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 296770832 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed15589887149568 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 182177532 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated312435838971904 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated296845821155328 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 182167862 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed311327496984576 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 114602970 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated296148371331072 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated296148371331072 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed15179142692864 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 _runtime7668.532801031 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 117383057 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 trainer/global_step17249 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed15179142692864 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current252 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 114602970 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated49463725 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15590017816576 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed67919582 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated15179230129664 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated311327601460736 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 296780504 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed49463475 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 Yu2*- _wandbruntime7668  _runtime7668gw* loss/FBt1/assignment_loss0 &loss/total_loss15.805465777409758 epoch0 trainer/global_step17249 /loss/FRt1/assignment_loss2.390701057943138 /loss/FRt2/assignment_loss2.343716319785061 % _timestamp1.7858029557111018e+09 _runtime7668.533936029 _step1379 0loss/SRqqt1/assignment_loss0.78291795795436 1loss/SRqqt2/assignment_loss1.077880068954381 /loss/FRt1/detection_loss2.5455221604253793 .loss/FRt2/detection_loss2.320190593510507 1loss/SRqqt1/detection_loss1.0838000408802477 -loss/FBt1/detection_loss1.33631892723497 /loss/FBt2/detection_loss0.8223220441257582 loss/FBt2/assignment_loss0 0loss/SRqqt2/detection_loss1.102096606595955⸻* loss/FBt1/assignment_loss0 &loss/total_loss15.805465777409758 epoch0 trainer/global_step17249 /loss/FRt1/assignment_loss2.390701057943138 /loss/FRt2/assignment_loss2.343716319785061 % _timestamp1.7858029557111018e+09 _runtime7668.533936029 _step1379 0loss/SRqqt1/assignment_loss0.78291795795436 1loss/SRqqt2/assignment_loss1.077880068954381 /loss/FRt1/detection_loss2.5455221604253793 .loss/FRt2/detection_loss2.320190593510507 1loss/SRqqt1/detection_loss1.0838000408802477 -loss/FBt1/detection_loss1.33631892723497 /loss/FBt2/detection_loss0.8223220441257582 loss/FBt2/assignment_loss0 0loss/SRqqt2/detection_loss1.102096606595955 hw*(: U gpu.0.gpu27.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp39.0 gpu.0.powerWatts104.933' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent34.97766666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0- Wc*(:  U disk.drbd1289.in0 disk.sda.in1.59375 disk.drbd1550.in0 disk.drbd1938.in0 disk.drbd1938.out0 disk.drbd1510.in0 cpu13.720980201413088 disk.drbd1584.in0 disk.drbd1552.out0 disk.drbd1289.out0% memory_percent41.263753175735474 disk.drbd1965.in0 disk.drbd1828.in0 disk.drbd1828.out0 disk.nvme0n1.in 4.64453125 disk.rbd0.in0 disk.drbd1510.out0 disk.drbd1356.out0 disk.drbd2041.out0 disk.sda1.in0 disk.sda2.out 210.6953125 disk.drbd1817.in0 network.sent 1.5865025e+07 network.recv854082$ disk./.usageGB578.3527183532715 disk.drbd2078.in0 disk.drbd1356.in0 disk.sda2.in1.59375 disk.drbd1226.out0 disk.dm-0.in 4.64453125! proc.memory.rssMB 11260.40625 disk.drbd2111.in0 disk.drbd1965.out0 disk.drbd1673.out0 disk.drbd1226.in0 disk.drbd1931.in0 disk.sda.out 210.6953125* proc.memory.availableMB38493.38671875 proc.cpu.threads29 disk.sda1.out0 disk.dm-0.out310.0361328125 disk.drbd2078.out0 disk.drbd1584.out0 disk.drbd2118.out0 disk.drbd2041.in0 disk.drbd1550.out0 disk.rbd0.out0 disk.drbd1285.out0* proc.memory.percent17.182016372680664 disk.drbd2118.in0 disk.rbd1.in0 disk.rbd1.out 30.84765625 disk.drbd1552.in0 disk.drbd1285.in0( disk./.usagePercent32.3445116587594 disk.drbd2111.out0 disk.drbd1817.out0 disk.drbd1673.in0 disk.drbd1931.out0# disk.nvme0n1.out310.0361328125& 2*- _wandbruntime7690  _runtime7690D* ' lr-AdamW/pg10.00044583305400931023 ' lr-AdamW/pg20.00044583305400931023 trainer/global_step17299 % _timestamp1.7858029774317565e+09 _runtime7690.254745932 _step1380* _runtime7690.254745932 _step1380 ' lr-AdamW/pg10.00044583305400931023 ' lr-AdamW/pg20.00044583305400931023 trainer/global_step17299 % _timestamp1.7858029774317565e+09 2*- _wandbruntime7690  _runtime7690뗓P!*O RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 117716733 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 _step1381 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed312211784887808 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 114928477 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed296989525679104 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current248 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed304794531063660 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current250 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 _runtime7690.259689361 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 182694997 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15222259208704 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated296989542718464 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated313323266733568 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed297688944974848 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 182685341 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 117716483 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed313323111155712 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed289609525813472 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed296989525679104 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated312211889356800 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed312211784887808 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 182685341 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated15222346638336 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated68112497 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 114928477 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 182694997 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated15634296855040 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 NEDeviceStatsMonitor.on_train_batch_start/allocÖation.small_pool.current9656 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 % _timestamp1.7858029774327426e+09 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed68112495 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed15634166180864 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed15185005250188 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 _step1381 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed312211784887808 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 114928477 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed296989525679104 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current248 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed304794531063660 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current250 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 _runtime7690.259689361 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 182694997 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15222259208704 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated296989542718464 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated313323266733568 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed297688944974848 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 182685341 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 117716483 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed313323111155712 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed289609525813472 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed296989525679104 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated312211889356800 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed312211784887808 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 182685341 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated15222346638336 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated68112497 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 114928477 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 182694997 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated15634296855040 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 % _timestamp1.7858029774327426e+09 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed68112495 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed15634166180864 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed15185005250188 Ўo2*- _wandbruntime7690  _runtime7690H&*M M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed15223138828288 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed304812148709202 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 297630984 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated15185968526646 trainer/global_step17299 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated312229935746560 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated304812251551798 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated15223226265088 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 117723231 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated68116434 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 114935122 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 297640656 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 182695864 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 182705534 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated297006709481472 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated297706176995328 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed15185882723410 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed312229831270400 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed289626265985792 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated15223226265088 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current235 _runtime7690.624276615 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated312229935746560 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed15223138828288 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed49606799 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current237 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 297640656 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed15635068040704 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 114935120 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed68116432 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated297006709481472 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 114935120 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed297706152091648 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed297006692442112 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _step1382 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 297630984 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 182705534 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed297006692442112 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed313341220132352 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated156352029020162"-&*M WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed297006692442112 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _step1382 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 297630984 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 182705534 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed297006692442112 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed313341220132352 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15635202902016 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed15223138828288 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed304812148709202 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 297630984 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated15185968526646 trainer/global_step17299 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated312229935746560 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated304812251551798 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated15223226265088 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 117723231 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated68116434 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 114935122 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 297640656 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 182695864 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 182705534 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated297006709481472 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated297706176995328 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed15185882723410 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed312229831270400 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed289626265985792 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated15223226265088 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current235 _runtime7690.624276615 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated312229935746560 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed15223138828288 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed49606799 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current237 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 297640656 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed15635068040704 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 114935120 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed68116432 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated297006709481472 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 114935120 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed297706152091648 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 L2*- _wandbruntime7690  _runtime7690* &loss/total_loss14.639384176392324 0loss/FRt2/assignment_loss2.0943850180682015 2loss/SRqqt2/assignment_loss0.9383824452238737 loss/FBt2/assignment_loss0 .loss/FRt1/detection_loss2.452773320674896 % _timestamp1.7858029778025525e+09 /loss/FRt1/assignment_loss2.260886956663693 trainer/global_step17299 _runtime7690.625383592 /loss/FBt2/detection_loss0.6079668506319004 2loss/SRqqt1/assignment_loss0.7534907914981802 3loss/FBt1/assignment_loss1.833988851383485e-08 .loss/FRt2/detection_loss2.298129619745647 1loss/SRqqt1/detection_loss1.0707958525410672 _step1383 1loss/SRqqt2/detection_loss1.0021087162074496 /loss/FBt1/detection_loss1.1604645867975285 epoch0 * ;v_runtime7690.625383592 /loss/FBt2/detection_loss0.6079668506319004 2loss/SRqqt1/assignment_loss0.7534907914981802 3loss/FBt1/assignment_loss1.833988851383485e-08 .loss/FRt2/detection_loss2.298129619745647 1loss/SRqqt1/detection_loss1.0707958525410672 _step1383 1loss/SRqqt2/detection_loss1.0021087162074496 /loss/FBt1/detection_loss1.1604645867975285 epoch0 &loss/total_loss14.639384176392324 0loss/FRt2/assignment_loss2.0943850180682015 2loss/SRqqt2/assignment_loss0.9383824452238737 loss/FBt2/assignment_loss0 .loss/FRt1/detection_loss2.452773320674896 % _timestamp1.7858029778025525e+09 /loss/FRt1/assignment_loss2.260886956663693 trainer/global_step17299 CS*(: U gpu.0.gpu29.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts96.608' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent32.202666666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0Ta*(:  ώU disk.drbd2078.out0$ memory_percent41.26430153846741* proc.memory.percent17.182016372680664 network.recv854148 disk.drbd1817.out0 disk.drbd1965.in0 disk.drbd1673.out0 disk.drbd2041.out0 disk.drbd1828.in0 disk.drbd1828.out0 disk.rbd0.in0 disk.drbd2078.in0 disk.drbd1584.out0 disk.drbd1817.in0 disk.drbd2041.in0* proc.memory.availableMB38493.02734375 proc.cpu.threads29 disk.drbd1285.out0 disk.drbd1289.in0 cpu13.720633343076466 disk.drbd1356.out0 disk.sda2.out 211.015625 disk.dm-0.out310.1884765625 network.sent 1.5865091e+07 disk.drbd1510.out0 disk.rbd1.in0 disk.drbd1965.out0 disk.drbd1938.in0 disk.sda2.in1.59375 disk.drbd2111.out0 disk.dm-0.in 4.64453125! proc.memory.rssMB 11260.40625 disk.drbd1931.in0 disk.drbd1938.out0 disk.rbd1.out 30.84765625 disk.sda1.in0 disk.drbd1226.in0 disk.drbd1356.in0 disk.drbd1552.out0 disk.drbd1550.out0 disk.rbd0.out0$ disk./.usageGB578.3526420593262 disk.drbd1289.out0) disk./.usagePercent32.34450739200258 disk.drbd1285.in0 disk.drbd2118.in0 disk.sda.out 211.015625 disk.drbd1584.in0 disk.drbd2118.out0# disk.nvme0n1.out310.1884765625 disk.drbd1673.in0 disk.drbd1931.out0 disk.sda.in1.59375 disk.sda1.out0 disk.drbd1226.out0 disk.drbd1552.in0 disk.drbd1550.in0 disk.drbd2111.in0 disk.nvme0n1.in 4.64453125 disk.drbd1510.in0u+*(: ТU gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts130.175' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent43.39166666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0mQMe*(:  U disk.drbd2041.out0 disk.sda1.in0 disk.drbd1510.in0 disk.drbd1510.out0 disk.dm-0.in 4.64453125* proc.memory.percent17.182016372680664 disk.drbd1673.in0 disk.drbd1817.in0 disk.drbd1552.out0 disk.drbd2118.out0) disk./.usagePercent32.34451059207019 disk.drbd1938.out0 disk.drbd1356.in0 disk.sda2.in1.59375 network.sent 1.5910175e+07$ disk./.usageGB578.3526992797852 disk.rbd1.out 30.91015625 disk.sda.out 211.36328125 disk.drbd1285.out0! proc.memory.rssMB 11260.40625 disk.drbd1965.out0 disk.rbd0.in0$ memory_percent41.26392602920532) proc.memory.availableMB 38493.2734375 disk.rbd1.in0 disk.sda.in1.59375 disk.drbd1226.out0 disk.drbd2118.in0 disk.drbd2111.out0 disk.sda1.out0 disk.rbd0.out0 network.recv855371 disk.drbd1938.in0 disk.drbd2078.in0 disk.drbd1584.in0 disk.drbd1356.out0 disk.drbd1828.out0! disk.dm-0.out310.50927734375 disk.drbd1550.out0 disk.drbd1226.in0 disk.nvme0n1.in 4.64453125 disk.drbd2041.in0 disk.drbd1552.in0 disk.drbd1584.out0 disk.drbd1285.in0 disk.drbd2078.out0 cpu13.72036888131532 disk.drbd1931.out0 disk.drbd1817.out0 disk.drbd1965.in0 disk.sda2.out 211.36328125 disk.drbd1289.in0 disk.drbd1289.out0 disk.drbd1550.in0 disk.drbd1931.in0 disk.drbd1828.in0$ disk.nvme0n1.out310.50927734375 proc.cpu.threads29 disk.drbd2111.in0 disk.drbd1673.out0rK2*- _wandbruntime7712  _runtime7712ݔՀ* _step1384 ' lr-AdamW/pg10.00044712166333357544 ' lr-AdamW/pg20.00044712166333357544 trainer/global_step17349 $ _timestamp1.785802999810562e+09 _runtime7712.633539693๱5* ' lr-AdamW/pg20.00044712166333357544 trainer/global_step17349 $ _timestamp1.785802999810562e+09 _runtime7712.633539693 _step1384 ' lr-AdamW/pg10.00044712166333357544 H2*- _wandbruntime7712  _runtime7712p@'*O YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated313114223642624 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 115260627 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 183222999 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed298549300814848 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated314228791374848 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 trainer/global_step17349 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 118057430 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated305675530985652 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated68309347 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed15679330787840 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 % _timestamp1.7858029998115296e+09 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated15266342773760 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15266255344128 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed313114119173632 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed290446534429472 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated49748332 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated290446551468832 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 115260627 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed313114119173632 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed290446534429472 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated49748332 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated290446551468832 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 115260627 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 118057430 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated305675530985652 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated68309347 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed15679330787840 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 % _timestamp1.7858029998115296e+09 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated15266342773760 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15266255344128 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 JDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 183233536 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated15267222400512 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed15267134963712 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated68313284 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated314246899017216 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 183223866 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed49750935 _step1386 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed313132165556224 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 118064217 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 298491136 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed297865030592512 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 _runtime7713.013931581 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 298500808 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15680366181888 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 115267270 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 115267272 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated305693148631306 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated313132270032384 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 118064475 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 298491136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated49751191 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated298566532835328 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed297865030592512 UHDeviceStatsMoni9.: tor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 115267272 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated297865047631872 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed15680237612032 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed15229771186918 % _timestamp1.7858030001905847e+09 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current256 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated15229856990154 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 183223866 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated313132270032384 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed15229771186918 % _timestamp1.7858030001905847e+09 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current256 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated15229856990154 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 183223866 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated313132270032384 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 183233536 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated15267222400512 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed15267134963712 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated68313284 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated314246899017216 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 183223866 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed49750935 _step1386 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed313132165556224 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 118064217 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 298491136 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed297865030592512 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 _runtime7713.013931581 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 298500808 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15680366181888 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 115267270 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 115267272 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated305693148631306 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated313132270032384 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 118064475 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 298491136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated49751191 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated298566532835328 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed297865030592512 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 115267272 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated297865047631872 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed15680237612032 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 EDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15310251479552 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 118398168 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 115592779 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 _step1389 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 299343780 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 299334122 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 118398426 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed68506195 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated298706219019264 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed314016453459456 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated49892229 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed315134213522944 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed15272782177204 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 115592777 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed299409656654848 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated291283560084832 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed291283543045472 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed314016453459456 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated298706219019264 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 % _timestamp1.7858030218638082e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed49891973 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 183741345 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121Fh"*O KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated314016557928448 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated291283560084832 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed291283543045472 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed314016453459456 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated298706219019264 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 % _timestamp1.7858030218638082e+09 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed49891973 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 183741345 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed298706201979904 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 115592779 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed15310251479552 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated314016557928448 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed298706201979904 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated68506197 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15310251479552 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 118398168 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 115592779 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 _step1389 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 299343780 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 299334122 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 118398426 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed68506195 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated298706219019264 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed314016453459456 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated49892229 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed315134213522944 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed15272782177204 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 115592777 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed299409656654848 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed291300283217792 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 183751868 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated291300300257152 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 $ _timestamp1.785803022242565e+09 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 299351288 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed15725454574592 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 183751868 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed315152318346240 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 118404909 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated15311218535936 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 183761538 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 118405155 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current244 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated298723385782272 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed298723368742912 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 _runtime7735.065907428 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed68510132 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15725593630208 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed306573942868218 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _step1390 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 299360960 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 163959296 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed15311131099136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current246 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 115599420 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated306574045710814 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed15273659650426 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated15273745453662 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed314034499842048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated299426888675328 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 139055616 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 299360960 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 115599420 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 183761538 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated49895021 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed15311131099136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated315152482305536 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 trainer/global_step17399 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated298723385782272 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 115599422 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated68510134 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 299360960 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 115599420 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 183761538 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated49895021 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed15311131099136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated315152482305536 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 trainer/global_step17399 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated298723385782272 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 115599422 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated68510134 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed291300283217792 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 183751868 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated291300300257152 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 $ _timestamp1.785803022242565e+09 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 299351288 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed15725454574592 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 183751868 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed315152318346240 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 118404909 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated15311218535936 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 183761538 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 118405155 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current244 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated298723385782272 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed298723368742912 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 _runtime7735.065907428 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed68510132 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15725593630208 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed306573942868218 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _step1390 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 299360960 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 163959296 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed15311131099136 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 v;@2*- _wandbruntime7735  _runtime7735zi* 1loss/SRqqt1/detection_loss1.1984257395379245 1loss/SRqqt2/detection_loss1.3303534132638224 epoch0 _step1391 &loss/total_loss15.122204570816415 trainer/global_step17399 % _timestamp1.7858030222441893e+09 0loss/FRt2/assignment_loss1.7148714829026983 2loss/SRqqt1/assignment_loss0.6363463672812366 .loss/FRt2/detection_loss2.278756127933438 _runtime7735.067009115 loss/FBt2/assignment_loss0 /loss/FRt1/detection_loss2.6774458784509347 /loss/FBt1/detection_loss1.1852476561272687 /loss/FBt2/detection_loss0.8120523875193864 /loss/FRt1/assignment_loss2.443969007279422 2loss/SRqqt2/assignment_loss0.8447364942644732 4loss/FBt1/assignment_loss1.6255810596600317e-08#Z* 4loss/FBt1/assignment_loss1.6255810596600317e-08 1loss/SRqqt1/detection_loss1.1984257395379245 1loss/SRqqt2/detection_loss1.3303534132638224 epoch0 _step1391 &loss/total_loss15.122204570816415 trainer/global_step17399 % _timestamp1.7858030222441893e+09 0loss/FRt2/assignment_loss1.7148714829026983 2loss/SRqqt1/assignment_loss0.6363463672812366 .loss/FRt2/detection_loss2.278756127933438 _runtime7735.067009115 loss/FBt2/assignment_loss0 /loss/FRt1/detection_loss2.6774458784509347 /loss/FBt1/detection_loss1.1852476561272687 /loss/FBt2/detection_loss0.8120523875193864 /loss/FRt1/assignment_loss2.443969007279422 2loss/SRqqt2/assignment_loss0.8447364942644732 x'*(: U gpu.0.gpu37.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts96.566' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent32.18866666666666 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0$7{ ^*(:  U* disk./.usagePercent32.344479871421115* proc.memory.percent17.182016372680664 proc.cpu.threads29 disk.drbd1285.out0 disk.drbd1510.out0 disk.drbd2078.out0 disk.rbd1.in0 disk.drbd2118.out0 disk.drbd2118.in0 disk.drbd2041.in0! proc.memory.rssMB 11260.40625 network.recv856877" disk.nvme0n1.out 311.005859375 disk.drbd1817.out0 disk.drbd1938.in0) proc.memory.availableMB 38492.3671875 disk.drbd1226.out0 disk.drbd1356.in0 disk.drbd1552.out0 disk.sda1.in0 disk.drbd1289.out0 disk.drbd1673.in0 disk.drbd1552.in0 disk.drbd1828.out0 disk.drbd1510.in0 disk.drbd2041.out0 disk.drbd1289.in0 disk.sda1.out0 disk.nvme0n1.in 4.64453125 disk.drbd2111.out0 disk.sda.in1.59375 disk.dm-0.out 311.005859375 disk.drbd1226.in0 disk.sda2.in1.59375 disk.sda2.out 212.21875 disk.drbd1817.in0 disk.drbd1931.in0 disk.drbd1828.in0 network.sent 1.595791e+07 disk.drbd1285.in0 disk.rbd1.out 31.0361328125 disk.drbd1550.out0 disk.drbd1938.out0 disk.rbd0.in0 cpu13.719730283764132 disk.drbd1584.in0 disk.drbd1584.out0 disk.drbd1550.in0$ memory_percent41.26530885696411$ disk./.usageGB578.3521499633789 disk.drbd1965.in0 disk.drbd1965.out0 disk.drbd1673.out0 disk.rbd0.out0 disk.sda.out 212.21875 disk.dm-0.in 4.64453125 disk.drbd2078.in0 disk.drbd1356.out0 disk.drbd2111.in0 disk.drbd1931.out0U*(: ʘU gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts125.566' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent41.855333333333334 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0g*(:  U disk.drbd1226.in0 disk.drbd2078.in0 disk.drbd1817.out0 disk.drbd1965.out0 disk.drbd1285.in0 disk.drbd1510.in0 disk.drbd1965.in0 disk.drbd1938.out0* proc.memory.availableMB38492.61328125 proc.cpu.threads29 disk.drbd1673.out0 disk.drbd1552.in0 disk.drbd2118.out0 disk.drbd1931.out0 disk.drbd2041.out0 disk.drbd1289.in0* proc.memory.percent17.182016372680664$ disk.nvme0n1.out316.57958984375 disk.drbd1226.out0 disk.drbd1584.in0 cpu13.719340301784998 network.recv858552* disk./.usagePercent32.344473044610204 disk.nvme0n1.in 4.64453125 disk.drbd1356.in0% memory_percent41.264933347702026$ disk./.usageGB578.3520278930664 disk.drbd1817.in0 disk.drbd1673.in0 disk.drbd1931.in0! proc.memory.rssMB 11260.40625 disk.rbd0.in0 disk.sda1.out0 disk.sda2.out 213.734375 disk.drbd1552.out0 disk.drbd2078.out0 disk.rbd1.in0 disk.rbd0.out0 disk.sda.in1.59375 disk.drbd2118.in0 disk.drbd2041.in0 disk.drbd2111.out0 disk.rbd1.out 31.0361328125 disk.drbd1828.in0 disk.dm-0.in 4.64453125 disk.drbd1828.out0 disk.sda.out 213.734375 disk.drbd1510.out0 disk.drbd1584.out0 disk.drbd2111.in0 disk.drbd1550.in0 disk.sda2.in1.59375 disk.drbd1550.out0 disk.drbd1285.out0! disk.dm-0.out316.57958984375 disk.drbd1289.out0 disk.drbd1938.in0 disk.drbd1356.out0 network.sent 1.6026548e+07 disk.sda1.in0-2*- _wandbruntime7756  _runtime7756[* & lr-AdamW/pg10.0004496988819821061 & lr-AdamW/pg20.0004496988819821061 trainer/global_step17449 % _timestamp1.7858030439198527e+09 _runtime7756.742810166 _step1392P* % _timestamp1.7858030439198527e+09 _runtime7756.742810166 _step1392 & lr-AdamW/pg10.0004496988819821061 & lr-AdamW/pg20.0004496988819821061 trainer/global_step17449 Bl*2*- _wandbruntime7756  _runtime7756(l *O UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 184279003 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated316039910564352 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 _runtime7756.744041353 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated68703047 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated15769873165824 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated307437325144668 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed307437222302184 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated299564557169664 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated15354335044608 KBDeviceStatsMonitor.on_train_batch_stayFP!rt/allocation.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed50034316 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 300194274 _step1393 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed68703045 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed15354247614976 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 184269347 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15354247614976 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current253 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed316039752889344 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed292120551661472 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 % _timestamp1.7858030439207857e+09 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current255 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 184269347 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated50034569 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed314918787745280 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated300270037398528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated15316756443836 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated15354335044608 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 115924929 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed299564540130304 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 184279003 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated299564557169664 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated314918892214272 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed15769740394496 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 trainer/global_step17449 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed15316670640712 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 115924929 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 115924927 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed299564540130304 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124G'*O \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed299564540130304 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 184279003 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated316039910564352 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 _runtime7756.744041353 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated68703047 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated15769873165824 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated307437325144668 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed307437222302184 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated299564557169664 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated15354335044608 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed50034316 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 300194274 _step1393 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed68703045 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed15354247614976 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 184269347 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15354247614976 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current253 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed316039752889344 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed292120551661472 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 % _timestamp1.7858030439207857e+09 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current255 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 184269347 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated50034569 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed314918787745280 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated300270037398528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated15316756443836 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated15354335044608 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 115924929 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed299564540130304 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 184279003 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated299564557169664 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated314918892214272 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed15769740394496 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 trainer/global_step17449 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed15316670640712 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 115924929 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 115924927 wk2*- _wandbruntime7757  _runtime7757LS&*M ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed307454839947726 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated15317633917170 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _step1394 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed314936834127872 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated314936938604032 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed68706982 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 115931572 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated307454942790322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated15355214671360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated50037437 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated292137308873152 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current240 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 115931570 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 184279870 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed15355127234560 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 184279870 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 300221112 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 115931570 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed316057865593344 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed292137291833792 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed299581706893312 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 184289540 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 115931572 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 118744421 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 300211440 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated314936938604032 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated15355214671360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 300221112 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 118744181 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed300287219611648 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed15770645981696/'[*M [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated299581723932672 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed15355127234560 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 184289540 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated300287244515328 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed68706982 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 115931572 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated307454942790322 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated15355214671360 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated50037437 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated292137308873152 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current240 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 115931570 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 184279870 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed15355127234560 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 184279870 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 300221112 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 115931570 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed316057865593344 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed292137291833792 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed299581706893312 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 184289540 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 115931572 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 118744421 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 300211440 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated314936938604032 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated15355214671360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 300221112 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 118744181 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed300287219611648 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed15770645981696 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed307454839947726 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated15317633917170 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 _step1394 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed314936834127872 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated314936938604032 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed68899895 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15398243750400 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated15815101621760 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated15360644907344 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 % _timestamp1.7858030658499568e+09 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated15398331180032 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated315821226500096 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed316945339282432 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated68899897 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 184797349 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed300422878280704 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 184797349 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 119078041 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current242 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 116257079 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated308318222224176 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 _step1397 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated316945494860288 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed15398243750400 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed315821122031104 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 119077797 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed301130368334848 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed308318119381692 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed15360559104220 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed315821122031104 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 _runtime7778.673143796 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 116257077 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed68899895 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15398243750400 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated15815101621760 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated15360644907344 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 % _timestamp1.7858030658499568e+09 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated15398331180032 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated315821226500096 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed316945339282432 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated68899897 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 184797349 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed300422878280704 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 184797349 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 119078041 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current242 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 116257079 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated308318222224176 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 _step1397 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated316945494860288 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed15398243750400 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed315821122031104 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 119077797 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed301130368334848 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed308318119381692 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed15360559104220 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed315821122031104 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 _runtime7778.673143796 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 116257077 J8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 301081264 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 116263722 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 301081264 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15816021931008 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 184807872 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 VIDeviceStatsMonitor.onIfP_train_batch_end/inactive_split.small_pool.allocated50180967 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed316963458327040 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated15399210806784 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated68903834 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 184817542 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed308335737027234 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 184817542 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 119084801 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 119084540 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed300440045043712 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current261 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed68903832 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated315839272889856 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed15399123369984 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed15399123369984 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed315839168413696 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 trainer/global_step17499 % _timestamp1.7858030662276204e+09 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated301147600355328 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated315839272889856 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed300440045043712 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated15361522380678 _step1398 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 116263720 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 301071592 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current259 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed15361436577442 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 116263720 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated316963622286336 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 116263722 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated300440062083072 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated315839272889856 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed300440045043712 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated15361522380678 _step1398 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 116263720 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 301071592 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current259 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed15361436577442 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 116263720 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated316963622286336 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 116263722 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated300440062083072 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 301081264 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 116263722 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 301081264 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15816021931008 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 184807872 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated50180967 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed316963458327040 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated15399210806784 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated68903834 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 184817542 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed308335737027234 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 184817542 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 119084801 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 119084540 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed300440045043712 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current261 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed68903832 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated315839272889856 L,2+- _wandbruntime7779  _runtime7779Y&!+ /loss/FRt2/assignment_loss2.160401139632765 2loss/SRqqt2/assignment_loss0.8524890823120819 1loss/SRqqt1/detection_loss1.3179970542100115 epoch0 loss/FBt1/assignment_loss0 -loss/FRt1/detection_loss2.43572680018414 1loss/SRqqt2/detection_loss0.9963643724252855 $ _timestamp1.785803066229296e+09 .loss/FRt2/detection_loss2.437178541917399 .loss/FBt1/detection_loss0.725854454934597 %loss/total_loss15.09744279943202 _runtime7779.052131892 _step1399 /loss/FRt1/assignment_loss2.251530110836029 2loss/SRqqt1/assignment_loss0.8397689068847349 loss/FBt2/assignment_loss0 /loss/FBt2/detection_loss1.0801323360949755 trainer/global_step17499oV!+ /loss/FRt2/assignment_loss2.160401139632765 2loss/SRqqt2/assignment_loss0.8524890823120819 1loss/SRqqt1/detection_loss1.3179970542100115 epoch0 loss/FBt1/assignment_loss0 -loss/FRt1/detection_loss2.43572680018414 1loss/SRqqt2/detection_loss0.9963643724252855 $ _timestamp1.785803066229296e+09 .loss/FRt2/detection_loss2.437178541917399 .loss/FBt1/detection_loss0.725854454934597 %loss/total_loss15.09744279943202 _runtime7779.052131892 _step1399 /loss/FRt1/assignment_loss2.251530110836029 2loss/SRqqt1/assignment_loss0.8397689068847349 loss/FBt2/assignment_loss0 /loss/FBt2/detection_loss1.0801323360949755 trainer/global_step17499 yf+(:  U disk.drbd1356.in0 disk.drbd1285.in0! proc.memory.rssMB 11260.40625 proc.cpu.threads29 disk.drbd1510.out0) proc.memory.availableMB 38493.0234375 disk.drbd1356.out0 network.recv859427 disk.nvme0n1.in 4.64453125 disk.rbd1.in0 cpu13.71864589810521 disk.drbd1584.out0 disk.drbd1552.in0 disk.drbd2118.out0$ disk.nvme0n1.out319.75439453125 disk.drbd1965.out0 disk.drbd1938.in0 disk.rbd0.in0* proc.memory.percent17.182016372680664 disk.drbd1931.out0 disk.drbd1828.out0 disk.rbd0.out0 disk.sda.in1.59375 disk.drbd2078.in0 disk.drbd1817.out0 disk.dm-0.in 4.64453125 disk.drbd1673.out0) disk./.usagePercent32.34453789931383 disk.sda2.in1.59375 disk.drbd2111.in0 disk.rbd1.out 31.0986328125 disk.drbd2078.out0$ disk./.usageGB578.3531875610352 disk.drbd2111.out0 disk.drbd1226.out0 disk.drbd1828.in0 disk.drbd1510.in0 disk.drbd1931.in0 disk.drbd1817.in0 disk.drbd1285.out0! disk.dm-0.out319.75439453125 disk.drbd2118.in0 disk.drbd1965.in0 disk.drbd1550.out0 disk.drbd1673.in0 disk.drbd1552.out0 disk.drbd1938.out0 disk.sda1.in0 disk.drbd1226.in0 disk.drbd1584.in0 disk.sda2.out 215.9296875 disk.drbd2041.out0 disk.drbd1289.in0 disk.sda.out 215.9296875% memory_percent41.264307498931885 disk.drbd2041.in0 disk.drbd1550.in0 disk.drbd1289.out0 disk.sda1.out0 network.sent 1.6029641e+07_+(: V gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts95.475' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent31.825 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0.`+(:  U disk.drbd1285.out0 network.recv859535 disk.drbd1226.out0 disk.rbd1.in0 disk.drbd1931.in0 disk.drbd1510.in0 disk.drbd2111.out0 disk.drbd1552.out0 network.sent 1.6029749e+07 disk.drbd1673.in0 disk.drbd1552.in0 disk.rbd0.in0 disk.rbd0.out0 proc.cpu.threads29 disk.drbd1584.in0 disk.drbd2118.in0 disk.drbd1673.out0 disk.drbd1550.out0 disk.drbd1965.in0 disk.drbd1285.in0$ disk./.usageGB578.3532409667969 disk.drbd1817.in0! proc.memory.rssMB 11260.40625 disk.drbd1965.out0 disk.drbd2041.in0 disk.drbd1938.in0 disk.drbd1817.out0 disk.drbd1828.out0 disk.drbd2078.out0 disk.sda2.in1.59375 disk.drbd2041.out0 disk.drbd1550.in0 disk.sda1.out0 disk.dm-0.in 4.64453125 disk.dm-0.out320.3798828125$ memory_percent41.26545190811157 disk.drbd1584.out0 disk.drbd2118.out0 disk.drbd1356.in0 disk.sda2.out 216.40625 disk.drbd1289.in0 cpu13.718419617975883# disk.nvme0n1.out320.3798828125 disk.rbd1.out 31.224609375 disk.drbd1289.out0 disk.sda1.in0* disk./.usagePercent32.344540886043596 disk.drbd1828.in0* proc.memory.percent17.182016372680664 disk.drbd1931.out0 disk.drbd1938.out0 disk.sda.out 216.40625 disk.drbd1510.out0) proc.memory.availableMB 38492.2734375 disk.drbd1226.in0 disk.drbd2078.in0 disk.nvme0n1.in 4.64453125 disk.drbd1356.out0 disk.drbd2111.in0 disk.sda.in1.59375/x+(: ԼU gpu.0.gpu25.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts95.983' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent31.994333333333337 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0ě2+- _wandbruntime7801  _runtime7801uO_6+ & lr-AdamW/pg10.0004522761006306367 & lr-AdamW/pg20.0004522761006306367 trainer/global_step17549 % _timestamp1.7858030884987288e+09 _runtime7801.323663763 _step1400sa+ & lr-AdamW/pg10.0004522761006306367 & lr-AdamW/pg20.0004522761006306367 trainer/global_step17549 % _timestamp1.7858030884987288e+09 _runtime7801.323663763 _step1400 X&o2+- _wandbruntime7801  _runtime7801p{!+O N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed316723456316928 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 119419211 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated301281233470464 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 185325351 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 134868480 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed293794568893472 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated301990749078528 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed301281216431104 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed15442239885824 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 301924236 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed301990724174848 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed50322194 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 301914578 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated50322464 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 185335007 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 116589227 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 trainer/global_step17549 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated316723560785920 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated15442327315456 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current270 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed15860114630656 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated309199119303684 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated317850998577664 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated293794585932832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed301281216431104 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed69096745 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed309199016461200 % _timestamp1.7858030884994006e+09 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed15404447567728 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated69096747 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed316723456316928 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated15442327315456 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 185325351 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed317850838805504 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 TGDeviceStatsMonitor.on_train_batch_starta?k/active_bytes.large_pool.current17039360 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 _runtime7801.323767628 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed301990724174848 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed50322194 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 301914578 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 159772160 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated50322464 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 185335007 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 116589227 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 trainer/global_step17549 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated316723560785920 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated15442327315456 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current270 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed15860114630656 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated309199119303684 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated317850998577664 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated293794585932832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed301281216431104 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed69096745 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed309199016461200 % _timestamp1.7858030884994006e+09 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed15404447567728 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated69096747 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed316723456316928 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated15442327315456 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 185325351 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed317850838805504 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 _runtime7801.323767628 JDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed316741502699520 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed301298383194112 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 185345544 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 185335874 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated293811326105152 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 _step1402 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current249 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed302007931291648 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 _runtime7801.704126894 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 301941416 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated15405410844186 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated69100684 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 301931744 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 116595872 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated302007956195328 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15861147395072 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed15443119505408 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed301298383194112 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 301931744 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed15443119505408 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated15443206942208 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated317869103590400 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated316741607175680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated301298400233472 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed309216634106742 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed317868948019712 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 % _timestamp1.7858030888807142e+09 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 116595870 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 185345544 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 119425690 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 301941416 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed15405325040950 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed293811309065792 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 130667008 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed316741502699520 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current251 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed301298383194112 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 185345544 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 185335874 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated293811326105152 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 _step1402 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current249 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed302007931291648 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 _runtime7801.704126894 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 301941416 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated15405410844186 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated69100684 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 301931744 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 116595872 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated302007956195328 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15861147395072 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed15443119505408 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed301298383194112 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 301931744 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed15443119505408 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated15443206942208 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated317869103590400 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated316741607175680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated301298400233472 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed309216634106742 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed317868948019712 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 % _timestamp1.7858030888807142e+09 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 116595870 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 185345544 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 119425690 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 301941416 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed15405325040950 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed293811309065792 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 [2+- _wandbruntime7801  _runtime7801+ 3loss/FBt2/assignment_loss9.536742027194124e-09 .loss/FRt1/detection_loss2.615362471342087 /loss/SRqqt2/detection_loss1.06978240140664 /loss/FBt1/detection_loss0.8927833297976145 &loss/total_loss15.718259212203863 epoch0 2loss/SRqqt2/assignment_loss0.8079325572555558 1loss/SRqqt1/detection_loss1.3443880417700191 2loss/SRqqt1/assignment_loss0.7110179568418451 .loss/FRt2/detection_loss2.253526122633661 /loss/FBt2/detection_loss1.1731436459952966 _runtime 7801.70532005 /loss/FRt1/assignment_loss2.591997266560793 /loss/FRt2/assignment_loss2.258296074492208 3loss/FBt1/assignment_loss2.933457140242904e-05 trainer/global_step17549 % _timestamp1.7858030888824182e+09 _step1403"+ /loss/FRt1/assignment_los]s2.591997266560793 /loss/FRt2/assignment_loss2.258296074492208 3loss/FBt1/assignment_loss2.933457140242904e-05 trainer/global_step17549 % _timestamp1.7858030888824182e+09 _step1403 3loss/FBt2/assignment_loss9.536742027194124e-09 .loss/FRt1/detection_loss2.615362471342087 /loss/SRqqt2/detection_loss1.06978240140664 /loss/FBt1/detection_loss0.8927833297976145 &loss/total_loss15.718259212203863 epoch0 2loss/SRqqt2/assignment_loss0.8079325572555558 1loss/SRqqt1/detection_loss1.3443880417700191 2loss/SRqqt1/assignment_loss0.7110179568418451 .loss/FRt2/detection_loss2.253526122633661 /loss/FBt2/detection_loss1.1731436459952966 _runtime 7801.70532005 p4e+(:  U disk.drbd1289.in0 proc.cpu.threads29 disk.drbd2041.out0 disk.drbd1285.out0) disk./.usagePercent32.34453811265167 disk.drbd2078.in0 disk.drbd2118.in0 disk.drbd2118.out0 disk.sda2.out 216.6328125* proc.memory.percent17.182016372680664 disk.drbd1552.in0 disk.drbd1510.out0 disk.sda2.in1.59375 cpu13.718098701006062 disk.dm-0.in 4.64453125 disk.drbd1965.in0 disk.drbd2041.in0 disk.drbd1550.out0 disk.drbd2078.out0 disk.drbd1828.in0 disk.drbd1584.out0 disk.drbd1817.out0 disk.drbd1673.in0 disk.drbd1931.in0 disk.sda.in1.59375 disk.drbd1817.in0 disk.drbd1828.out0 disk.drbd1550.in0# disk.nvme0n1.out320.6962890625 disk.drbd1356.in0* proc.memory.availableMB38492.62890625 disk.drbd1931.out0 disk.sda1.out0 disk.dm-0.out320.6962890625 disk.drbd1226.out0 disk.drbd1356.out0 network.recv861071 disk.drbd1938.in0 disk.drbd1285.in0 disk.drbd2111.in0! proc.memory.rssMB 11260.40625 disk.drbd1938.out0 disk.rbd0.in0 disk.drbd1289.out0 disk.drbd1965.out0 disk.sda.out 216.6328125% memory_percent41.264909505844116$ disk./.usageGB578.3531913757324 disk.rbd0.out0 disk.drbd1510.in0 disk.nvme0n1.in 4.64453125 disk.drbd2111.out0 disk.rbd1.in0 disk.rbd1.out 31.224609375 disk.drbd1673.out0 disk.drbd1552.out0 disk.sda1.in0 disk.drbd1226.in0 disk.drbd1584.in0 network.sent 1.6098311e+07&+(: ĺU gpu.0.gpu0.0 gpu.0.memory0, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts127.755' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent42.585 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0mljv2+- _wandbruntime7823  _runtime7823z + _step1404 ' lr-AdamW/pg10.00045356470995490205 ' lr-AdamW/pg20.00045356470995490205 trainer/global_step17599 % _timestamp1.7858031106316724e+09 _runtime7823.454633754+ ' lr-AdamW/pg20.00045356470995490205 trainer/global_step17599 % _timestamp1.7858031106316724e+09 _runtime7823.454633754 _step1404 ' lr-AdamW/pg10.00045356470995490205 )z2+- _wandbruntime7823  _runtime7823p'+O UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed317625790602752 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 % _timestamp1.7858031106325727e+09 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated15448421834360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated294631594548832 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 _runtime7823.457694127 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed50466070 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed310079913540708 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed302139554581504 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 119759904 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated50466307 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated15905416435200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 116921379 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed318756369970176 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 185853353 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed317625790602752 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated302139571620864 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated318756521353728 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated69293597 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 _step1405 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated15486323450880 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15486236021248 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 185853353 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed302851080014848 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 116921379 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed318756369970176 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 185853353 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed317625790602752 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated302139571620864 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated318756521353728 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated69293597 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 _step1405 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated15486323450880 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15486236021248 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 185853353 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed302851080014848 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 % _timestamp1.7858031106325727e+09 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 151383552 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated15448421834360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated294631594548832 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 _runtime7823.457694127 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed50466070 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed310079913540708 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed302139554581504 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 119759904 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated50466307 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated15905416435200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 JDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 % _timestamp1.7858031110145254e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed318774474547712 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed15449213504458 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 185873546 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 116928022 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated15487203077632 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 302801568 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 119766403 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15906324374528 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed294648317681792 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 302791896 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 119766656 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated310097634028846 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed50468871 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed302868287131648 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated294648334721152 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated15449299307694 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated318774636409856 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed310097531186250 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _step1406 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed302156721344512 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated302868312035328 _runtime7823.837860964 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated15487203077632 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed310097531186250 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _step1406 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed302156721344512 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated302868312035328 _runtime7823.837860964 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated15487203077632 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 % _timestamp1.7858031110145254e+09 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed318774474547712 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed15449213504458 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 185873546 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 116928022 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated15487203077632 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 302801568 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 119766403 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15906324374528 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed294648317681792 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 302791896 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 119766656 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated310097634028846 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed50468871 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed302868287131648 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated294648334721152 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated15449299307694 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 Pϱ2+- _wandbruntime7823  _runtime7823Ky + 1loss/SRqqt1/assignment_loss0.947429407216675 2loss/SRqqt2/assignment_loss0.9010814363484516 1loss/SRqqt1/detection_loss1.2601258843338916 /loss/FBt1/detection_loss1.7378842028390085 /loss/FBt2/detection_loss0.8831032777670771 3loss/FBt2/assignment_loss2.662330786051825e-07 .loss/FRt2/detection_loss2.522327935269901 trainer/global_step17599 _runtime 7823.83894981 0loss/FRt2/assignment_loss2.4045734763145448 /loss/FRt1/detection_loss2.6285835606487176 % _timestamp1.7858031110161304e+09 _step1407 3loss/FBt1/assignment_loss5.253631498090629e-06 1loss/SRqqt2/detection_loss0.9742334504715271 &loss/total_loss16.759828998997747 epoch0 /loss/FRt1/assignment_loss2.500480847923379#6+ 0loss/FRt2/assignment_loss2.4045734763145448 /loss/FRt1/detection_loss2.6285835606487176 % _timestamp1.7858031110161304e+09 _step1407 3loss/FBt1/assignment_loss5.253631498090629e-06 1loss/SRqqt2/detection_loss0.9742334504715271 &loss/total_loss16.759828998997747 epoch0 /loss/FRt1/assignment_loss2.500480847923379 1loss/SRqqt1/assignment_loss0.947429407216675 2loss/SRqqt2/assignment_loss0.9010814363484516 1loss/SRqqt1/detection_loss1.2601258843338916 /loss/FBt1/detection_loss1.7378842028390085 /loss/FBt2/detection_loss0.8831032777670771 3loss/FBt2/assignment_loss2.662330786051825e-07 .loss/FRt2/detection_loss2.522327935269901 trainer/global_step17599 _runtime 7823.83894981 ra+(:  U disk.sda2.out 217.515625* proc.memory.percent17.182016372680664 disk.drbd1828.out0 disk.rbd0.in0 disk.drbd1226.in0 network.recv861137) disk./.usagePercent32.34453811265167 disk.drbd1931.in0 disk.sda.out 217.515625 disk.drbd2078.out0 disk.drbd1356.in0 disk.drbd1965.out0 disk.drbd2118.in0$ disk.nvme0n1.out321.28662109375 disk.drbd1817.in0 disk.drbd1817.out0 proc.cpu.threads29 disk.drbd1289.in0 disk.dm-0.in 4.64453125 disk.drbd2078.in0 disk.drbd2111.out0 disk.rbd1.in0 disk.drbd1965.in0 disk.drbd2041.in0 disk.drbd1552.in0 disk.rbd1.out 31.373046875$ disk./.usageGB578.3531913757324 disk.sda2.in1.59375' proc.memory.availableMB 38492.84375 disk.drbd1285.out0! disk.dm-0.out321.28662109375 disk.drbd1828.in0 disk.sda.in1.59375 disk.drbd1584.out0 disk.drbd2111.in0 disk.drbd1289.out0 disk.drbd1510.in0 disk.drbd2118.out0$ memory_percent41.26458168029785 disk.sda1.out0 disk.drbd1673.in0 disk.drbd1673.out0 disk.drbd1931.out0 disk.drbd2041.out0 disk.drbd1550.out0 disk.drbd1938.in0 disk.drbd1584.in0 disk.drbd1552.out0 disk.drbd1550.in0 disk.rbd0.out0 disk.drbd1510.out0 disk.drbd1226.out0 disk.nvme0n1.in 4.64453125! proc.memory.rssMB 11260.40625 cpu13.717795146111635 disk.drbd1938.out0 disk.sda1.in0 disk.drbd1285.in0 disk.drbd1356.out0 network.sent 1.6098377e+07L"+(: ՍV gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts104.607' gpu.0.enforcedPowerLimitWatts300.0 gpu.0.powerPercent34.869 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0yeQ^+(: ڧU gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts130.127' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent43.37566666666667 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0p+h+(:  U disk.drbd1510.in0 disk.drbd1226.in0 disk.drbd2078.out0% memory_percent41.265708208084106! proc.memory.rssMB 11260.40625 disk.sda.in1.59375$ disk.nvme0n1.out321.43896484375 disk.drbd1356.out0 disk.sda2.out 217.80078125 disk.drbd1931.in0 disk.drbd1552.out0 disk.drbd1938.out0! disk.dm-0.out321.43896484375 disk.rbd1.out 31.373046875 disk.drbd1828.in0 disk.drbd1550.out0 disk.drbd1938.in0 disk.drbd2118.out0 disk.drbd1965.out0 network.sent 1.6104035e+07 disk.drbd2078.in0 network.recv862135 disk.drbd1289.out0 disk.drbd1285.in0 disk.drbd1226.out0 disk.drbd1584.in0 disk.drbd2118.in0 disk.drbd1931.out0 disk.drbd2041.out0 disk.drbd1552.in0 disk.sda.out 217.80078125 disk.drbd1817.in0* proc.memory.availableMB38492.10546875 disk.drbd1673.in0 disk.drbd2041.in0 disk.drbd1510.out0 disk.dm-0.in 4.64453125 disk.drbd1584.out0 disk.rbd0.out0 disk.drbd1356.in0 disk.drbd2111.in0 disk.drbd1817.out0 proc.cpu.threads29$ disk./.usageGB578.3531532287598 disk.drbd1550.in0 disk.drbd1289.in0 disk.sda2.in1.59375 disk.drbd2111.out0 disk.drbd1965.in0 disk.drbd1673.out0* proc.memory.percent17.182016372680664 disk.sda1.in0 disk.nvme0n1.in 4.64453125 disk.drbd1828.out0 disk.rbd1.in0 disk.rbd0.in0 disk.sda1.out0 cpu13.71750928773773) disk./.usagePercent32.34453597927325 disk.drbd1285.out0{\2+- _wandbruntime7846  _runtime78465e90+ trainer/global_step17649 $ _timestamp1.785803133244187e+09 _runtime7846.067149078 _step1408 ' lr-AdamW/pg10.00045485331927916736 ' lr-AdamW/pg20.00045485331927916736W+ ' lr-AdamW/pg10.00045485331927916736 ' lr-AdamW/pg20.00045485331927916736 trainer/global_step17649 $ _timestamp1.785803133244187e+09 _runtime7846.067149078 _step1408 2+- _wandbruntime7846  _runtime7846/t'+O YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated318528229357568 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated310960913462700 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed50609357 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 303644540 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed302997892731904 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed15492224494744 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated15492310297868 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed15530232156672 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed318528124888576 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated303711460758528 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated319662072868864 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed303711435854848 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed295468586125472 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated295468603164832 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated15530319586304 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 186391011 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _step1409 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 117253529 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed15950481436160 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 trainer/global_step17649 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated302997909771264 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 % _timestamp1.7858031332451441e+09 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 186381355 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 117253527 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 120099802 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated15530319586304 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 186391011 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current264 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed69490445 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed319661917291008 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed310960810620216 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed318528124888576 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated15950612110336 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 303634882 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 120100066 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated318528229357568 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15530232156672 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 186381355 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 117253527 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed302997892731904 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated69490447 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated302997909771264 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current262 Jtc+O E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed319661917291008 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed310960810620216 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed318528124888576 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated15950612110336 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 303634882 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 120100066 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated318528229357568 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15530232156672 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 186381355 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 117253527 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed302997892731904 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated69490447 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated302997909771264 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current262 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed302997892731904 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed15492224494744 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated15492310297868 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed15530232156672 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed318528124888576 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated303711460758528 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated319662072868864 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed303711435854848 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed295468586125472 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated295468603164832 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated15530319586304 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 186391011 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 _step1409 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 117253529 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed15950481436160 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 trainer/global_step17649 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated302997909771264 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 % _timestamp1.7858031332451441e+09 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 186381355 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 117253527 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 120099802 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated15530319586304 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 186391011 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current264 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed69490445 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 a2+- _wandbruntime7846  _runtime7846j&+M SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _runtime7846.458112708 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated50612484 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 120106868 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed310978428265758 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated303015076534272 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 117260170 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed303015059494912 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 303661720 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 120106608 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current260 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated295485343337152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 186401548 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated69494384 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed15493101967966 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15951516076544 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed50612226 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 186391878 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated303015076534272 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed303015059494912 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 186391878 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated310978531108354 % _timestamp1.7858031336346486e+09 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 117260172 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed69494382 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144&+M OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 117260170 _step1410 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated303728667875328 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 186401548 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated69494384 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed15493101967966 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated15951516076544 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed50612226 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 186391878 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated303015076534272 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed303015059494912 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 186391878 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated310978531108354 % _timestamp1.7858031336346486e+09 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 117260172 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed69494382 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _runtime7846.458112708 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated50612484 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 120106868 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed310978428265758 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated303015076534272 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 117260170 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed303015059494912 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 303661720 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 120106608 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current260 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated295485343337152 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 PDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed319430459174400 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 186919013 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.pea!ak11816 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 186909357 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 186909357 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 _step1413 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 117585677 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated50753753 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated296305611780832 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated319430563643392 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 155577856 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed15536112958252 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed303856230882304 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 304504692 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed311841707699724 % _timestamp1.7858031557347965e+09 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 117585677 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated303856247921664 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated311841810542208 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed320567500502528 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current265 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed319430459174400 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current267 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 304495034 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated15995839481856 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed296305594741472 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated15574315721728 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15574228292096 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 130674176 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated297142620396832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 117917829 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 120783152 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed320332793460224 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current274 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current272 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated320332897929216 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 _step1417 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 117917827 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed297142603357472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated304714586072064 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated312722707621716 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed15618224427520 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed304714569032704 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed15580001421760 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated304714586072064 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated320332897929216 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated50899005 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed50898733 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed305432147534848 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated69884147 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated321473223322624 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated305432172438528 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated16041050884096 _runtime 7891.03984323 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 305355186 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 120782878|'+O MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 117917829 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 161869312 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 120783152 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed320332793460224 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current274 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current272 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated320332897929216 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 _step1417 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 117917827 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed297142603357472 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated304714586072064 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 136965632 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated312722707621716 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed15618224427520 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed304714569032704 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed15580001421760 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated304714586072064 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated320332897929216 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated50899005 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed50898733 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed305432147534848 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated69884147 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated321473223322624 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated305432172438528 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated16041050884096 _runtime 7891.03984323 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 305355186 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 120782878 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated15618311857152 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15618224427520 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed69884145 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 187437359 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed304714569032704 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 305364844 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 187447015 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated15618311857152 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed321473061453312 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 trainer/global_step17749 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 187437359 # _timestamp1.78580317821658e+09 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 117917827 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated297142620396832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 52+- _wandbruntime7891  _runtime7891tf+M F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed312740222424774 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 305372352 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed321491175216128 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 187457552 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed320350839842816 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated16041949134336 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 117924472 KDDeviceStatsMonitor.on_trainz_batch_end/inactive_split.large_pool.peak18 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated305449379555328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 187447882 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated15580964698218 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 120789978 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 305372352 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed50901634 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed15619104047104 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 117924472 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 305382024 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current262 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated69888084 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated15619191483904 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated304731752835072 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed16041820564480 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 305382024 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed69888082 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 trainer/global_step17749 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated312740325267370 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed304731735795712 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 _runtime7891.427890276 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed15619104047104 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 120789716 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 % _timestamp1.7858031786043887e+09 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated15619191483904 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392;R&+M >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 % _timestamp1.7858031786043887e+09 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated15619191483904 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed312740222424774 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 305372352 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed321491175216128 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 128569856 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 187457552 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed320350839842816 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated16041949134336 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 117924472 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated305449379555328 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 187447882 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated15580964698218 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 120789978 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 305372352 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed50901634 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed15619104047104 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 117924472 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 305382024 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current262 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated69888084 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated15619191483904 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated304731752835072 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed16041820564480 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 305382024 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed69888082 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 trainer/global_step17749 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated312740325267370 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed304731735795712 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 153473536 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 _runtime7891.427890276 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed15619104047104 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 120789716 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 a2+- _wandbruntime7891  _runtime7891g+ _runtime7891.429551369 /loss/FRt1/assignment_loss2.358337620678155 1loss/SRqqt2/detection_loss1.0182775950601295 /loss/FBt1/detection_loss1.2303289968128268 2loss/SRqqt2/assignment_loss0.6134510631878196 .loss/FRt2/detection_loss2.565487554073334 /loss/FBt2/detection_loss1.2259059438171487 epoch0 _step1419 3loss/FBt1/assignment_loss5.960434731976324e-07 /loss/FRt1/detection_loss2.4759619190641073 0loss/SRqqt1/detection_loss1.192153345754134 $ _timestamp1.785803178606674e+09 /loss/FRt2/assignment_loss2.153764458497365 2loss/SRqqt1/assignment_loss0.7359656225455413 2loss/FBt2/assignment_loss0.001455096336994662 %loss/total_loss15.57108981187103 trainer/global_step17749iR + .loss/FRt2/detection_loss2.565487554073334 /loss/FBt2/detection_loss1.2259059438171487 epoch0 _step1419 3loss/FBt1/assignment_loss5.960434731976324e-07 /loss/FRt1/detection_loss2.4759619190641073 0loss/SRqqt1/detection_loss1.192153345754134 $ _timestamp1.785803178606674e+09 /loss/FRt2/assignment_loss2.153764458497365 2loss/SRqqt1/assignment_loss0.7359656225455413 2loss/FBt2/assignment_loss0.001455096336994662 %loss/total_loss15.57108981187103 trainer/global_step17749 _runtime7891.429551369 /loss/FRt1/assignment_loss2.358337620678155 1loss/SRqqt2/detection_loss1.0182775950601295 /loss/FBt1/detection_loss1.2303289968128268 2loss/SRqqt2/assignment_loss0.6134510631878196 _{+(: U gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts103.991' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent34.663666666666664 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors07og+(:  U$ disk.nvme0n1.out322.70458984375 disk.drbd1550.out0 disk.drbd2118.in0 disk.drbd1817.in0 disk.drbd1965.in0 disk.rbd0.in0 disk.drbd1510.in0 disk.drbd1226.out0( proc.memory.availableMB 38492.140625 disk.drbd2111.out0 disk.drbd1965.out0 disk.sda.out 219.46484375 disk.drbd1285.out0 disk.drbd1673.out0 disk.drbd1289.out0 disk.sda1.in0! disk.dm-0.out322.70458984375 disk.drbd1226.in0 disk.drbd2078.out0 disk.drbd1584.out0 disk.drbd1356.in0 disk.drbd1931.out0 disk.drbd2078.in0$ memory_percent41.26565456390381 disk.sda2.in1.59375 disk.rbd1.out 31.5615234375 disk.drbd1550.in0 disk.drbd1510.out0 disk.drbd1356.out0 disk.drbd1931.in0 disk.drbd2041.out0 disk.drbd1938.in0 disk.dm-0.in 4.64453125 disk.drbd2041.in0 disk.drbd1938.out0 disk.sda.in1.59375 disk.sda1.out0 disk.drbd2118.out0 disk.nvme0n1.in 4.64453125 disk.drbd1817.out0 disk.drbd1828.out0 disk.drbd1285.in0 disk.drbd1584.in0 disk.rbd1.in0 disk.drbd1552.in0 disk.rbd0.out0 disk.sda2.out 219.46484375 disk.drbd1673.in0# disk./.usageGB578.353157043457 disk.drbd1552.out0 cpu13.716342245551903 network.sent 1.6287081e+07 disk.drbd1828.in0! proc.memory.rssMB 11260.40625 network.recv867831* proc.memory.percent17.182016372680664 proc.cpu.threads29* disk./.usagePercent32.344536192611095 disk.drbd2111.in0 disk.drbd1289.in02M 2+- _wandbruntime7914  _runtime7914\u+ _runtime7914.003856458 _step1420 ' lr-AdamW/pg10.00045871914725196324 ' lr-AdamW/pg20.00045871914725196324 trainer/global_step17799 % _timestamp1.7858032011808007e+09{+ ' lr-AdamW/pg10.00045871914725196324 ' lr-AdamW/pg20.00045871914725196324 trainer/global_step17799 % _timestamp1.7858032011808007e+09 _runtime7914.003856458 _step1420 ԧ2+- _wandbruntime7914  _runtime7914f^'+O N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 141159936 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 187975017 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 187965361 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed16086128157696 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed321235127746048 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15662220562944 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 306224996 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated306292528278528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed313603501858740 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated322378797596160 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed306292503374848 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed321235127746048 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated297979629012832 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated321235232215040 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated305572924222464 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 118249979 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 121123259 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 187965361 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed51042001 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed15623889885268 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated70080997 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 % _timestamp1.7858032011818564e+09 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed305572907183104 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated15662307992576 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 trainer/global_step17799 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated305572924222464 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated16086269317632 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current263 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed70080995 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current261 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 121122996 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated313603604701224 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated15662307992576 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed15662220562944 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 187975017 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18v+O `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed16086128157696 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments@F'.freed0 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed321235127746048 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15662220562944 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 306224996 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated306292528278528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed313603501858740 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated322378797596160 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed306292503374848 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed321235127746048 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated297979629012832 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated321235232215040 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated305572924222464 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 118249979 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 121123259 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 187965361 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed51042001 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed15623889885268 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated70080997 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 % _timestamp1.7858032011818564e+09 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed305572907183104 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated15662307992576 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 trainer/global_step17799 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated305572924222464 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated16086269317632 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current263 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed70080995 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current261 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 121122996 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated313603604701224 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated15662307992576 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed15662220562944 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 187975017 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 141159936 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 187975017 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 187965361 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 2+- _wandbruntime7914  _runtime7914C6&+M K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 118256620 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed15663100182528 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current232 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 118256622 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed322396739452416 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated313621222346878 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 121129782 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed70084932 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated51045082 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 187985554 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 187985554 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed313621119504282 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed305590073946112 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 306242176 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated70084934 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 306232504 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated305590090985472 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 % _timestamp1.7858032015666122e+09 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 187975884 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current234 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated15663187619328 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2>&+M PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 121130016 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated297996369185152 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated15624853161726 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 187975884 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed321253174128640 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated15663187619328 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 306242176 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated305590090985472 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 118256620 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed51044850 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed305590073946112 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed15663100182528 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 187985554 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 187985554 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed313621119504282 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed305590073946112 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 306242176 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated70084934 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 306232504 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated305590090985472 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 % _timestamp1.7858032015666122e+09 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 187975884 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current234 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated15663187619328 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 118256620 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed15663100182528 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current232 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 118256622 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed322396739452416 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated313621222346878 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 121129782 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed70084932 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated51045082 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 P+(: V gpu.0.gpu26.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts99.553' gpu.0.enforcedPowerLimitWatts300.0) gpu.0.powerPercent33.184333333333335 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0[2+- _wandbruntime7936  _runtime7936X+ trainer/global_step17849 $ _timestamp1.785803223698425e+09 _runtime7936.521389925 _step1424 & lr-AdamW/pg10.0004600077565762285 & lr-AdamW/pg20.0004600077565762285Z@N+ _runtime7936.521389925 _step1424 & lr-AdamW/pg10.0004600077565762285 & lr-AdamW/pg20.0004600077565762285 trainer/global_step17849 $ _timestamp1.785803223698425e+09 7$O2+- _wandbruntime7936  _runtime7936e '+O QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 188503019 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 188503019 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed307152859214848 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed298816620589472 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed15667778348776 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 _step1425 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated306431262372864 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed15706216698368 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed322137462031872 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 188493363 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 307085148 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated314484501780732 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 118582129 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated322137566500864 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current253 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed70277845 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed322137462031872 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 118582129 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated15667864151900 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 118582127 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated51185861 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated15706304128000 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated323284352687616 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed306431245333504 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15706216698368 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 166063616 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 _runtime7936.522694721 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated322137566500864 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed323284186624000 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated70277847 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 % _timestamp1.7858032236993873e+09 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 307075490 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated15667864151900 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 118582127 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak548 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated51185861 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated15706304128000 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated323284352687616 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed306431245333504 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15706216698368 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 166063616 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 _runtime7936.522694721 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated322137566500864 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed323284186624000 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated70277847 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak251 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 % _timestamp1.7858032236993873e+09 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 307075490 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 188493363 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 307085148 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated314484501780732 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 118582129 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated322137566500864 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current253 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed70277845 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed322137462031872 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 118582129 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 l2+- _wandbruntime7936  _runtime7936!+M OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 188513556 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 188503886 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 118588772 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed323302293691904 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed306448412096512 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 121470456 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated322155612890624 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 188503886 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed51188426 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 trainer/global_step17849 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed298833360761792 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current246 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed307170066331648 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated70281784 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed15707096317952 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 307102328 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime7936.910345108 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed15668655821998 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 118588770 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated306448429135872 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated51188672 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 307092656 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 118588770 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated15707183754752 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 118588772 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated307170091235328 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed322155508414464 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 307092656 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated298833377801152 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 121470208 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated314502119426386\&+M ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed306448412096512 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated15707183754752 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 _runtime7936.910345108 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed15668655821998 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 118588770 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated306448429135872 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated51188672 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 307092656 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 118588770 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 136958464 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated15707183754752 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 118588772 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated307170091235328 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed322155508414464 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 307092656 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated298833377801152 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 121470208 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated314502119426386 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 188513556 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 188503886 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 118588772 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed323302293691904 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed306448412096512 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 121470456 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated322155612890624 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 188503886 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed51188426 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 161862144 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 trainer/global_step17849 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed298833360761792 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current246 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed307170066331648 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated70281784 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed15707096317952 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 307102328 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 189031021 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 118914279 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated324189951284224 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated323039900786688 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated15711752615408 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 _step1429 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 189021365 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated299653646244832 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 _runtime7958.970679215 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed307289583483904 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15750212833792 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 189031021 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 trainer/global_step17899 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 118914279 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed308013215054848 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated323039900786688 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed323039796317696 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed307289583483904 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed323039796317696 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed324189797803520 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated323039900786688 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed323039796317696 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed307289583483904 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed323039796317696 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed324189797803520 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 189031021 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 UGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated 118914279 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated324189951284224 ZLDeviceStatsMonitoQL r.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated323039900786688 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak 250523136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated15711752615408 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 _step1429 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 QCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed 189021365 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 ZKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated 2768240640 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated299653646244832 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 _runtime7958.970679215 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed307289583483904 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15750212833792 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 189031021 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 trainer/global_step17899 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 118914279 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed308013215054848 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated15751179890176 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed16177482093568 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated299670386417152 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed323057842700288 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed51332309 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed324207904265216 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated323057947176448 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated324208061933056 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed70478632 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed299670369377792 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed307306750246912 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 118920922 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated51332562 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated323057947176448 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 118920920 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 118920922 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed15751092453376 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated315383016505894 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 189031888 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 307962480 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 189041558 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 121810941 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated70478634 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed308030422171648 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated16177614857728 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed315382913663298 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated307306767286272 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated15712630088742 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current253 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 121811196 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 189041558 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed15751092453376 _step1430 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 118920920 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated308030447075328 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed15712544285506 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 trainer/global_step17899 _runtime7959.351708926 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 307952808 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated307306767286272 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 189031888 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 2537553928&,M ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated15751179890176 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated307306767286272 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated15712630088742 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current253 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 157667840 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 121811196 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 189041558 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed15751092453376 _step1430 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 118920920 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated308030447075328 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed15712544285506 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 trainer/global_step17899 _runtime7959.351708926 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 307952808 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated307306767286272 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 189031888 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed307306750246912 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated15751179890176 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed16177482093568 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated299670386417152 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed323057842700288 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed51332309 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed324207904265216 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated323057947176448 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated324208061933056 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed70478632 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed299670369377792 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed307306750246912 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 118920922 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated51332562 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated323057947176448 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 118920920 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 118920922 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed15751092453376 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated315383016505894 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 189031888 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 307962480 SEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated 189041558 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 121810941 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated70478634 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed308030422171648 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated16177614857728 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed315382913663298 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 PDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current259 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 trainer/global_step17949 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 119246429 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed323942130603520 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed51473117 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated308873595798528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 189559023 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated15755641078916 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated323942235072512 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 119246427 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 189559023 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 122144662 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current257 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated325095508928512 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated308147938673664 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed16221784552960 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 308795794 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed300490637821472 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated323942235072512^&',O G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 M?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed 189549367 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed15755555275792 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak11816 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed70671545 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated316246295939748 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 L=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak 2940455936 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current259 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 trainer/global_step17949 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 119246429 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed323942130603520 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 153480704 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed0 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed51473117 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated308873595798528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 QCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated 189559023 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated15755641078916 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 128577024 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated323942235072512 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 119246427 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 189559023 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 122144662 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current257 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated325095508928512 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated308147938673664 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed16221784552960 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 308795794 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed300490637821472 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated323942235072512 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed308147921634304 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 % _timestamp1.7858032686135504e+09 RDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated 122144921 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated323960281462272 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 119253070 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed15795088588800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed316263810742806 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 189559890 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated16222818546176 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated300507395033152 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated308165105436672 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 119253072 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated325113621461504 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated15795176025600 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 119253070 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed70675482 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 _step1434 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed308165088397312 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed308890778011648 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed15756432749014 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated323960281462272 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated15795176025600 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated51476280 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed325113461696512 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated15756518552250 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _runtime7981.819109697 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed15795088588800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current254 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 122151510 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed300507377993792 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 % _timestamp1.7858032689957554e+09 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed323960176986112 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated70675484 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 308812960 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 119253072 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 122151764 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 308822632 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 189569560 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 308822632 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current252 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated316263913585402 trainer/global_step17949 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated308165105436672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 308812960 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed16222683684864 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 189559890 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236<,M =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries0 TEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak 2645753856 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 XIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated 2768240640 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak11887 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed325113461696512 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated15756518552250 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current121 _runtime7981.819109697 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed15795088588800 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current254 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 122151510 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed300507377993792 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current130 % _timestamp1.7858032689957554e+09 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed323960176986112 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak548 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated70675484 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current 104476160 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed 308812960 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak0 OADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated 119253072 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current2 PBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated 122151764 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 308822632 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 KDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak18 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 134861312 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 189569560 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 308822632 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current252 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated316263913585402 trainer/global_step17949 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated308165105436672 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed 308812960 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed16222683684864 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated251 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 189559890 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak 145651200 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current85803236 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated308890802915328 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 QBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated 3021996032 O@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current 3021996032 RDDeviceStatsMonitor.on_tra6gPin_batch_end/reserved_bytes.small_pool.peak 253755392 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current87436800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed51476028 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated121 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated323960281462272 L=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak 3021996032 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak130 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 119253070 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed15795088588800 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak 250523136 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed316263810742806 OADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed 189559890 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated16222818546176 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated300507395033152 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated308165105436672 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak1291 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current24903680 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 119253072 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated325113621461504 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated15795176025600 OADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed 119253070 4.DeviceStatsMonitor.on_train_batch_end/num_ooms0 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current 253755392 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed70675482 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated0 _step1434 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed308165088397312 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed308890778011648 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed15756432749014 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated323960281462272 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 159764992 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated15795176025600 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak11816 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated51476280 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current0 -2,- _wandbruntime7981  _runtime7981g, 3loss/FBt1/assignment_loss1.868856627353921e-05 4loss/FBt2/assignment_loss1.9423918235408084e-06 _step1435 0loss/FRt2/assignment_loss2.2877575421617147 /loss/FRt2/detection_loss2.4149635525509003 1loss/SRqqt2/detection_loss1.1446695048575297 /loss/FRt1/assignment_loss2.241827857958806 2loss/SRqqt1/assignment_loss0.6884618623932409 /loss/FRt1/detection_loss2.3886389620118327 /loss/FBt2/detection_loss1.0632362959988635 &loss/total_loss15.170994623576167 epoch0 trainer/global_step17949 2loss/SRqqt2/assignment_loss0.9542249322332129 1loss/SRqqt1/detection_loss0.9609288656049304 /loss/FBt1/detection_loss1.0262646168470382 % _timestamp1.7858032689973843e+09 _runtime7981.820213657?, trainer/global_step17949 2loss/SRqqt2/assignment_loss0.9542249322332129 1loss/SRqqt1/detection_loss0.9609288656049304 /loss/FBt1/detection_loss1.0262646168470382 % _timestamp1.7858032689973843e+09 _runtime7981.820213657 3loss/FBt1/assignment_loss1.868856627353921e-05 4loss/FBt2/assignment_loss1.9423918235408084e-06 _step1435 0loss/FRt2/assignment_loss2.2877575421617147 /loss/FRt2/detection_loss2.4149635525509003 1loss/SRqqt2/detection_loss1.1446695048575297 /loss/FRt1/assignment_loss2.241827857958806 2loss/SRqqt1/assignment_loss0.6884618623932409 /loss/FRt1/detection_loss2.3886389620118327 /loss/FBt2/detection_loss1.0632362959988635 &loss/total_loss15.170994623576167 epoch0 Х,(: U gpu.0.gpu39.0 gpu.0.memory2, gpu.0.memoryAllocated4.1587066650390625) gpu.0.memoryAllocatedBytes 3572301824 gpu.0.temp38.0 gpu.0.powerWatts98.377' gpu.0.enforcedPowerLimitWatts300.0( gpu.0.powerPercent32.79233333333333 gpu.0.smClock1410 gpu.0.memoryClock1512! gpu.0.correctedMemoryErrors0# gpu.0.uncorrectedMemoryErrors0ek,(:  U disk.drbd1931.in0 disk.drbd1550.out0 disk.drbd1938.out0 disk.drbd1285.out0 proc.cpu.threads29 disk.drbd1673.in0! disk.dm-0.out324.14892578125 disk.sda1.out0 disk.drbd1584.in0* proc.memory.percent17.182016372680664 network.recv873623 disk.rbd1.in0 disk.drbd1965.out0 disk.drbd1289.out0 disk.drbd2118.out0 disk.drbd1817.in0 disk.dm-0.in 4.64453125 disk.drbd1584.out0 disk.drbd1931.out0 disk.drbd1552.out0 disk.drbd1550.in0 disk.drbd2041.out0 disk.drbd1828.out0 disk.drbd1938.in0 disk.sda1.in0 disk.drbd1356.in0$ disk./.usageGB578.3530807495117 disk.drbd1226.out0 disk.sda2.out 221.76171875 disk.drbd2111.out0* proc.memory.availableMB38492.19921875* disk./.usagePercent32.344531925854284 disk.rbd1.out 31.9619140625 disk.drbd1289.in0 disk.rbd0.in0! proc.memory.rssMB 11260.40625 cpu13.714516128339467 disk.sda.in1.59375 disk.drbd1510.out0 disk.drbd1828.in0 disk.nvme0n1.in 4.64453125 disk.drbd2111.in0 network.sent 1.6472623e+07 disk.drbd1673.out0 disk.sda.out 221.76171875 disk.drbd1510.in0 disk.drbd2078.out0% memory_percent41.265565156936646 disk.drbd1285.in0 disk.drbd1226.in0 disk.drbd2078.in0 disk.drbd2118.in0 disk.drbd1356.out0 disk.drbd1817.out0 disk.drbd1965.in0 disk.drbd1552.in0 disk.rbd0.out0 disk.sda2.in1.59375 disk.drbd2041.in0$ disk.nvme0n1.out324.14892578125kyW2,- _wandbruntime8004  _runtime8004߼, _runtime8004.241102962 _step1436 ' lr-AdamW/pg10.00046387358454902443 ' lr-AdamW/pg20.00046387358454902443 trainer/global_step17999 % _timestamp1.7858032914180999e+09oV*, ' lr-AdamW/pg10.00046387358454902443 ' lr-AdamW/pg20.00046387358454902443 trainer/global_step17999 % _timestamp1.7858032914180999e+09 _runtime8004.241102962 _step1436 L5)2,- _wandbruntime8004  _runtime8004Z',O SDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak 2712961024 O@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak 2940455936 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current87429632 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current 157675008 =6DeviceStatsMonitor.on_train_batch_start/max_split_size-1 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated 253755392 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed15838205104640 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries0 N?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak 3021996032 XIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current 2768240640 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated15838292534272 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed309733926734848 60DeviceStatsMonitor.on_train_batch_start/num_ooms0 N@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed 122485703 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak538 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated326001096664576 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current 253755392 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current87429632 MFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak18 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current 132771328 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current17039360 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current 104468992 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated0 QBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current 3021996032 % _timestamp1.7858032914191172e+09 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed324844464889344 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated317127193019256 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed70868395 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated324844569358336 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 190087025 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated309006276824064 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current248 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated70868397 trainer/global_step17999 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed16267012254720 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated16267145026048 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 119578579 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed324844464889344 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed309006259784704 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed317127090176772 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed309006259784704 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 119578577 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 309655946 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15838205104640 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated51617554 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 _runtime8004.242350767 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated15838292534272 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated309733951638528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 _step1437 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed326000938989568 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated324844569358336 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 309665604 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 119578577 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed324844464889344 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current9658 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak1291 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated317127193019256 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak11816 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed70868395 UFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak 2768240640 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated324844569358336 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current17039360 UGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated 190087025 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current251 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak11887 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated309006276824064 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current248 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated70868397 trainer/global_step17999 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current2 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed16267012254720 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current130 C:DeviceStatsMonitor.on_train_batch_start/active.all.current9658 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak11887 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated16267145026048 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc251 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections0 QCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated 119578579 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed324844464889344 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed309006259784704 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current24903680 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed317127090176772 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated130 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current 104468992 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed309006259784704 QCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed 119578577 SEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak 189288448 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed 309655946 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed15838205104640 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated51617554 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak1291 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current9656 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current2 O@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak 2871874888 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed0 _runtime8004.242350767 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated121 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated15838292534272 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated309733951638528 YLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak86278144 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed0 _step1437 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak 145651200 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak121 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current17039360 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed326000938989568 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current 102842484 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated324844569358336 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated 309665604 SDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated 3021996032 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current121 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current85803124 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak 248886712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak 253755392 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated251 M?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed 119578577 =7DeviceStatsMonitor.on_train_batch_start/num_device_free0 VGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak 2645753856 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current9656 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak 250523136 VGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak 2712961024 JDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 309682784 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed309023426547712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 309682784 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 190087892 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated15839172161024 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated324862615748096 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 119585222 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed51620240 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed326019068200448 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 trainer/global_step17999 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated309751158755328 VGDeviceStatsMonitor.on_Ńtrain_batch_end/reserved_bytes.large_pool.current 2768240640 $ _timestamp1.785803291803881e+09 _step1438 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated309023443587072 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 190097562 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 122492572 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 151376384 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 126472704 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed16267934348800 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated324862615748096 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 119585220 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated51620485 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed15839084724224 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current247 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc251 K=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed 119585220 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak 250523136 TEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak 2712961024 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated51620485 M>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak 2871874888 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed15839084724224 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current247 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated 309682784 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak11887 M>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current87436800 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak251 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed309023426547712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak 248886712 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated 309682784 K=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed 190087892 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current 104476160 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated15839172161024 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated324862615748096 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current17039360 SEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated 119585222 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak538 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed51620240 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed326019068200448 WJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak86278144 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current 102842596 SDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak 2768240640 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak121 trainer/global_step17999 A8DeviceStatsMonitor.on_train_batch_end/active.all.current9672 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current9670 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated309751158755328 VGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current 2768240640 $ _timestamp1.785803291803881e+09 _step1438 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated309023443587072 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated 253755392 OADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated 190097562 J;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak 2940455936 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current17039360 L>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed 122492572 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak 253755392 QBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak 2712961024 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current 151376384 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current2 QCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak 189288448 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current 126472704 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed16267934348800 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current251 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated324862615748096 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current9670 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size-1 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed0 P", line 198, in _run_module_as_main I, qvnjquy8j  ǔ)P, qvnjquy8j> ژ)/ File "", line 88, in _run_code o0, qvnjquy8j  )CIҤ, qvnjquy8j 譟) File "/opt/conda/lib/python3.12/site-packages/spanet/train.py", line 277, in main(**parser.parse_args().__dict__) %rH, qvnjquy8j  褢)]!, qvnjquy8j ) File "/opt/conda/lib/python3.12/site-packages/spanet/train.py", line 207, in main trainer.fit(model, ckpt_path=checkpoint) /h, qvnjquy8j  К), qvnjquy8j ) File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/trainer/trainer.py", line 584, in fit call._call_and_handle_interrupt( 5, qvnjquy8j  )qr, qvnjquy8j ) File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/trainer/call.py", line 49, in _call_and_handle_interrupt return trainer_fn(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^ , qvnjquy8j  ) {, qvnjquy8j ໷) File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/trainer/trainer.py", line 630, in _fit_impl self._run(model, ckpt_path=ckpt_path, weights_only=weights_only) T,, qvnjquy8j  Һ)M\, qvnjquy8j ) File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/trainer/trainer.py", line 1079, in _run results = self._run_stage() ^^^^^^^^^^^^^^^^^ 5{, qvnjquy8j  )dN, qvnjquy8j ) File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/trainer/trainer.py", line 1123, in _run_stage self.fit_loop.run() iN=H, qvnjquy8j  )v, qvnjquy8j )z File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/loops/fit_loop.py", line 217, in run self.advance() l, qvnjquy8j  );, qvnjquy8j ) File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/loops/fit_loop.py", line 469, in advance self.epoch_loop.run(self._data_fetcher) j6, qvnjquy8j  ) z], qvnjquy8j ؁) File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 153, in run self.advance(data_fetcher) %s, qvnjquy8j  )7S, qvnjquy8j ) File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 352, in advance batch_output = self.automatic_optimization.run(trainer.optimizers[0], batch_idx, kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ PP, qvnjquy8j  )="L&, qvnjquy8j ) File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 192, in run self._optimizer_step(batch_idx, closure) ͣ, qvnjquy8j  )!, qvnjquy8j ) File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 270, in _optimizer_step call._call_lightning_module_hook( 9,, qvnjquy8j  ),g, qvnjquy8j ) File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/trainer/call.py", line 177, in _call_lightning_module_hook output = fn(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^ &(G, qvnjquy8j  Ӓ* ^, qvnjquy8j * File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/core/module.py", line 1368, in optimizer_step optimizer.step(closure=optimizer_closure) )t, qvnjquy8j  *';3, qvnjquy8j * File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/core/optimizer.py", line 154, in step step_output = self._strategy.optimizer_step(self._optimizer, closure, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ V7, qvnjquy8j  *5`, qvnjquy8j * File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/strategies/strategy.py", line 239, in optimizer_step return self.precision_plugin.optimizer_step(optimizer, model=model, closure=closure, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ mݱ, qvnjquy8j  Ԥ*z , qvnjquy8j * File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/plugins/precision/precision.py", line 123, in optimizer_step return optimizer.step(closure=closure, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ?JP, qvnjquy8j  *'V, qvnjquy8j * File "/opt/conda/lib/python3.12/site-packages/torch/optim/lr_scheduler.py", line 166, in wrapper return func.__get__(opt, opt.__class__)(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ', qvnjquy8j  *1, qvnjquy8j * File "/opt/conda/lib/python3.12/site-packages/torch/optim/optimizer.py", line 530, in wrapper out = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ d , qvnjquy8j  *.I, qvnjquy8j ̾* File "/opt/conda/lib/python3.12/site-packages/torch/optim/optimizer.py", line 80, in _use_grad ret = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ , qvnjquy8j  *Ѭe&, qvnjquy8j * File "/opt/conda/lib/python3.12/site-packages/torch/optim/adam.py", line 227, in step loss = closure() ^^^^^^^^^ {4, qvnjquy8j  б*F, qvnjquy8j * File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/plugins/precision/precision.py", line 109, in _wrap_closure closure_result = closure() ^^^^^^^^^ fEm, qvnjquy8j  *n, qvnjquy8j * File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 146, in __call__ self._result = self.closure(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ \, qvnjquy8j  *, qvnjquy8j * File "/opt/conda/lib/python3.12/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context return func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ G"0, qvnjquy8j  *w, qvnjquy8j * File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 131, in closure step_output = self._step_fn() ^^^^^^^^^^^^^^^ /^, qvnjquy8j  *^zc, qvnjquy8j * File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 319, in _training_step training_step_output = call._call_strategy_hook(trainer, "training_step", *kwargs.values()) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ , qvnjquy8j  *pK, qvnjquy8j * File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/trainer/call.py", line 329, in _call_strategy_hook output = fn(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^ 5, qvnjquy8j  *^g, qvnjquy8j * File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/strategies/strategy.py", line 391, in training_step return self.lightning_module.training_step(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ $Ѕt, qvnjquy8j  *+, qvnjquy8j Ъ* File "/opt/conda/lib/python3.12/site-packages/spanet/network/jet_reconstruction/jet_reconstruction_training.py", line 265, in training_step raise ValueError("Assignment targets contain a collision.") B{, qvnjquy8j  С*HU, qvnjquy8jC *4ValueError: Assignment targets contain a collision. I6k, qvnjquy8j  *?2,- _wandbruntime8011  _runtime8011=,(>